

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <meta name="Description" content="scikit-learn: machine learning in Python">

  
  <title>1.11. Ensemble methods &mdash; scikit-learn 0.22 documentation</title>
  
  <link rel="canonical" href="http://scikit-learn.org/stable/modules/ensemble.html" />

  
  <link rel="shortcut icon" href="../_static/favicon.ico"/>
  

  <link rel="stylesheet" href="../_static/css/vendor/bootstrap.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<script id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
<script src="../_static/jquery.js"></script> 
</head>
<body>
<nav id="navbar" class="sk-docs-navbar navbar navbar-expand-md navbar-light bg-light py-0">
  <div class="container-fluid sk-docs-container px-0">
      <a class="navbar-brand py-0" href="../index.html">
        <img
          class="sk-brand-img"
          src="../_static/scikit-learn-logo-small.png"
          alt="logo"/>
      </a>
    <button
      id="sk-navbar-toggler"
      class="navbar-toggler"
      type="button"
      data-toggle="collapse"
      data-target="#navbarSupportedContent"
      aria-controls="navbarSupportedContent"
      aria-expanded="false"
      aria-label="Toggle navigation"
    >
      <span class="navbar-toggler-icon"></span>
    </button>

    <div class="sk-navbar-collapse collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav mr-auto">
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../install.html">Install</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../user_guide.html">User Guide</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="classes.html">API</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../auto_examples/index.html">Examples</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../getting_started.html">Getting Started</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../tutorial/index.html">Tutorial</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../glossary.html">Glossary</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../developers/index.html">Development</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../faq.html">FAQ</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../related_projects.html">Related packages</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../roadmap.html">Roadmap</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../about.html">About us</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://github.com/scikit-learn/scikit-learn">GitHub</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://scikit-learn.org/dev/versions.html">Other Versions</a>
        </li>
        <li class="nav-item dropdown nav-more-item-dropdown">
          <a class="sk-nav-link nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
          <div class="dropdown-menu" aria-labelledby="navbarDropdown">
              <a class="sk-nav-dropdown-item dropdown-item" href="../getting_started.html">Getting Started</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../tutorial/index.html">Tutorial</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../glossary.html">Glossary</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../developers/index.html">Development</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../faq.html">FAQ</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../related_projects.html">Related packages</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../roadmap.html">Roadmap</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../about.html">About us</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://github.com/scikit-learn/scikit-learn">GitHub</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://scikit-learn.org/dev/versions.html">Other Versions</a>
          </div>
        </li>
      </ul>
      <div id="searchbox" role="search">
          <div class="searchformwrapper">
          <form class="search" action="../search.html" method="get">
            <input class="sk-search-text-input" type="text" name="q" aria-labelledby="searchlabel" />
            <input class="sk-search-text-btn" type="submit" value="Go" />
          </form>
          </div>
      </div>
    </div>
  </div>
</nav>
<div class="d-flex" id="sk-doc-wrapper">
    <input type="checkbox" name="sk-toggle-checkbox" id="sk-toggle-checkbox">
    <label id="sk-sidemenu-toggle" class="sk-btn-toggle-toc btn sk-btn-primary" for="sk-toggle-checkbox">Toggle Menu</label>
    <div id="sk-sidebar-wrapper" class="border-right">
      <div class="sk-sidebar-toc-wrapper">
        <div class="sk-sidebar-toc-logo">
          <a href="../index.html">
            <img
              class="sk-brand-img"
              src="../_static/scikit-learn-logo-small.png"
              alt="logo"/>
          </a>
        </div>
        <div class="btn-group w-100 mb-2" role="group" aria-label="rellinks">
            <a href="tree.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="1.10. Decision Trees">Prev</a><a href="../supervised_learning.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="1. Supervised learning">Up</a>
            <a href="multiclass.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="1.12. Multiclass and multilabel algorithms">Next</a>
        </div>
        <div class="alert alert-danger p-1 mb-2" role="alert">
          <p class="text-center mb-0">
          <strong>scikit-learn 0.22</strong><br/>
          <a href="http://scikit-learn.org/dev/versions.html">Other versions</a>
          </p>
        </div>
        <div class="alert alert-warning p-1 mb-2" role="alert">
          <p class="text-center mb-0">
            Please <a class="font-weight-bold" href="../about.html#citing-scikit-learn"><string>cite us</string></a> if you use the software.
          </p>
        </div>
          <div class="sk-sidebar-toc">
            <ul>
<li><a class="reference internal" href="#">1.11. Ensemble methods</a><ul>
<li><a class="reference internal" href="#bagging-meta-estimator">1.11.1. Bagging meta-estimator</a></li>
<li><a class="reference internal" href="#forests-of-randomized-trees">1.11.2. Forests of randomized trees</a><ul>
<li><a class="reference internal" href="#random-forests">1.11.2.1. Random Forests</a></li>
<li><a class="reference internal" href="#extremely-randomized-trees">1.11.2.2. Extremely Randomized Trees</a></li>
<li><a class="reference internal" href="#parameters">1.11.2.3. Parameters</a></li>
<li><a class="reference internal" href="#parallelization">1.11.2.4. Parallelization</a></li>
<li><a class="reference internal" href="#feature-importance-evaluation">1.11.2.5. Feature importance evaluation</a></li>
<li><a class="reference internal" href="#totally-random-trees-embedding">1.11.2.6. Totally Random Trees Embedding</a></li>
</ul>
</li>
<li><a class="reference internal" href="#adaboost">1.11.3. AdaBoost</a><ul>
<li><a class="reference internal" href="#usage">1.11.3.1. Usage</a></li>
</ul>
</li>
<li><a class="reference internal" href="#gradient-tree-boosting">1.11.4. Gradient Tree Boosting</a><ul>
<li><a class="reference internal" href="#classification">1.11.4.1. Classification</a></li>
<li><a class="reference internal" href="#regression">1.11.4.2. Regression</a></li>
<li><a class="reference internal" href="#fitting-additional-weak-learners">1.11.4.3. Fitting additional weak-learners</a></li>
<li><a class="reference internal" href="#controlling-the-tree-size">1.11.4.4. Controlling the tree size</a></li>
<li><a class="reference internal" href="#mathematical-formulation">1.11.4.5. Mathematical formulation</a><ul>
<li><a class="reference internal" href="#loss-functions">1.11.4.5.1. Loss Functions</a></li>
</ul>
</li>
<li><a class="reference internal" href="#regularization">1.11.4.6. Regularization</a><ul>
<li><a class="reference internal" href="#shrinkage">1.11.4.6.1. Shrinkage</a></li>
<li><a class="reference internal" href="#subsampling">1.11.4.6.2. Subsampling</a></li>
</ul>
</li>
<li><a class="reference internal" href="#interpretation">1.11.4.7. Interpretation</a><ul>
<li><a class="reference internal" href="#feature-importance">1.11.4.7.1. Feature importance</a></li>
</ul>
</li>
</ul>
</li>
<li><a class="reference internal" href="#histogram-based-gradient-boosting">1.11.5. Histogram-Based Gradient Boosting</a><ul>
<li><a class="reference internal" href="#id23">1.11.5.1. Usage</a></li>
<li><a class="reference internal" href="#missing-values-support">1.11.5.2. Missing values support</a></li>
<li><a class="reference internal" href="#low-level-parallelism">1.11.5.3. Low-level parallelism</a></li>
<li><a class="reference internal" href="#why-it-s-faster">1.11.5.4. Why it’s faster</a></li>
</ul>
</li>
<li><a class="reference internal" href="#voting-classifier">1.11.6. Voting Classifier</a><ul>
<li><a class="reference internal" href="#majority-class-labels-majority-hard-voting">1.11.6.1. Majority Class Labels (Majority/Hard Voting)</a><ul>
<li><a class="reference internal" href="#id26">1.11.6.1.1. Usage</a></li>
</ul>
</li>
<li><a class="reference internal" href="#weighted-average-probabilities-soft-voting">1.11.6.2. Weighted Average Probabilities (Soft Voting)</a></li>
<li><a class="reference internal" href="#using-the-votingclassifier-with-gridsearchcv">1.11.6.3. Using the <code class="docutils literal notranslate"><span class="pre">VotingClassifier</span></code> with <code class="docutils literal notranslate"><span class="pre">GridSearchCV</span></code></a><ul>
<li><a class="reference internal" href="#id27">1.11.6.3.1. Usage</a></li>
</ul>
</li>
</ul>
</li>
<li><a class="reference internal" href="#voting-regressor">1.11.7. Voting Regressor</a><ul>
<li><a class="reference internal" href="#id29">1.11.7.1. Usage</a></li>
</ul>
</li>
<li><a class="reference internal" href="#stacked-generalization">1.11.8. Stacked generalization</a></li>
</ul>
</li>
</ul>

          </div>
      </div>
    </div>
    <div id="sk-page-content-wrapper">
      <div class="sk-page-content container-fluid body px-md-3" role="main">
        
  <div class="section" id="ensemble-methods">
<span id="ensemble"></span><h1>1.11. Ensemble methods<a class="headerlink" href="#ensemble-methods" title="Permalink to this headline">¶</a></h1>
<p>The goal of <strong>ensemble methods</strong> is to combine the predictions of several
base estimators built with a given learning algorithm in order to improve
generalizability / robustness over a single estimator.</p>
<p>Two families of ensemble methods are usually distinguished:</p>
<ul>
<li><p>In <strong>averaging methods</strong>, the driving principle is to build several
estimators independently and then to average their predictions. On average,
the combined estimator is usually better than any of the single base
estimator because its variance is reduced.</p>
<p><strong>Examples:</strong> <a class="reference internal" href="#bagging"><span class="std std-ref">Bagging methods</span></a>, <a class="reference internal" href="#forest"><span class="std std-ref">Forests of randomized trees</span></a>, …</p>
</li>
<li><p>By contrast, in <strong>boosting methods</strong>, base estimators are built sequentially
and one tries to reduce the bias of the combined estimator. The motivation is
to combine several weak models to produce a powerful ensemble.</p>
<p><strong>Examples:</strong> <a class="reference internal" href="#adaboost"><span class="std std-ref">AdaBoost</span></a>, <a class="reference internal" href="#gradient-boosting"><span class="std std-ref">Gradient Tree Boosting</span></a>, …</p>
</li>
</ul>
<div class="section" id="bagging-meta-estimator">
<span id="bagging"></span><h2>1.11.1. Bagging meta-estimator<a class="headerlink" href="#bagging-meta-estimator" title="Permalink to this headline">¶</a></h2>
<p>In ensemble algorithms, bagging methods form a class of algorithms which build
several instances of a black-box estimator on random subsets of the original
training set and then aggregate their individual predictions to form a final
prediction. These methods are used as a way to reduce the variance of a base
estimator (e.g., a decision tree), by introducing randomization into its
construction procedure and then making an ensemble out of it. In many cases,
bagging methods constitute a very simple way to improve with respect to a
single model, without making it necessary to adapt the underlying base
algorithm. As they provide a way to reduce overfitting, bagging methods work
best with strong and complex models (e.g., fully developed decision trees), in
contrast with boosting methods which usually work best with weak models (e.g.,
shallow decision trees).</p>
<p>Bagging methods come in many flavours but mostly differ from each other by the
way they draw random subsets of the training set:</p>
<blockquote>
<div><ul class="simple">
<li><p>When random subsets of the dataset are drawn as random subsets of the
samples, then this algorithm is known as Pasting <a class="reference internal" href="#b1999" id="id1"><span>[B1999]</span></a>.</p></li>
<li><p>When samples are drawn with replacement, then the method is known as
Bagging <a class="reference internal" href="#b1996" id="id2"><span>[B1996]</span></a>.</p></li>
<li><p>When random subsets of the dataset are drawn as random subsets of
the features, then the method is known as Random Subspaces <a class="reference internal" href="#h1998" id="id3"><span>[H1998]</span></a>.</p></li>
<li><p>Finally, when base estimators are built on subsets of both samples and
features, then the method is known as Random Patches <a class="reference internal" href="#lg2012" id="id4"><span>[LG2012]</span></a>.</p></li>
</ul>
</div></blockquote>
<p>In scikit-learn, bagging methods are offered as a unified
<a class="reference internal" href="generated/sklearn.ensemble.BaggingClassifier.html#sklearn.ensemble.BaggingClassifier" title="sklearn.ensemble.BaggingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">BaggingClassifier</span></code></a> meta-estimator  (resp. <a class="reference internal" href="generated/sklearn.ensemble.BaggingRegressor.html#sklearn.ensemble.BaggingRegressor" title="sklearn.ensemble.BaggingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">BaggingRegressor</span></code></a>),
taking as input a user-specified base estimator along with parameters
specifying the strategy to draw random subsets. In particular, <code class="docutils literal notranslate"><span class="pre">max_samples</span></code>
and <code class="docutils literal notranslate"><span class="pre">max_features</span></code> control the size of the subsets (in terms of samples and
features), while <code class="docutils literal notranslate"><span class="pre">bootstrap</span></code> and <code class="docutils literal notranslate"><span class="pre">bootstrap_features</span></code> control whether
samples and features are drawn with or without replacement. When using a subset
of the available samples the generalization accuracy can be estimated with the
out-of-bag samples by setting <code class="docutils literal notranslate"><span class="pre">oob_score=True</span></code>. As an example, the
snippet below illustrates how to instantiate a bagging ensemble of
<code class="xref py py-class docutils literal notranslate"><span class="pre">KNeighborsClassifier</span></code> base estimators, each built on random subsets of
50% of the samples and 50% of the features.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">BaggingClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.neighbors</span> <span class="kn">import</span> <span class="n">KNeighborsClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">bagging</span> <span class="o">=</span> <span class="n">BaggingClassifier</span><span class="p">(</span><span class="n">KNeighborsClassifier</span><span class="p">(),</span>
<span class="gp">... </span>                            <span class="n">max_samples</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">max_features</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_bias_variance.html#sphx-glr-auto-examples-ensemble-plot-bias-variance-py"><span class="std std-ref">Single estimator versus bagging: bias-variance decomposition</span></a></p></li>
</ul>
</div>
<div class="topic">
<p class="topic-title">References</p>
<dl class="citation">
<dt class="label" id="b1999"><span class="brackets"><a class="fn-backref" href="#id1">B1999</a></span></dt>
<dd><p>L. Breiman, “Pasting small votes for classification in large
databases and on-line”, Machine Learning, 36(1), 85-103, 1999.</p>
</dd>
<dt class="label" id="b1996"><span class="brackets"><a class="fn-backref" href="#id2">B1996</a></span></dt>
<dd><p>L. Breiman, “Bagging predictors”, Machine Learning, 24(2),
123-140, 1996.</p>
</dd>
<dt class="label" id="h1998"><span class="brackets"><a class="fn-backref" href="#id3">H1998</a></span></dt>
<dd><p>T. Ho, “The random subspace method for constructing decision
forests”, Pattern Analysis and Machine Intelligence, 20(8), 832-844,
1998.</p>
</dd>
<dt class="label" id="lg2012"><span class="brackets"><a class="fn-backref" href="#id4">LG2012</a></span></dt>
<dd><p>G. Louppe and P. Geurts, “Ensembles on Random Patches”,
Machine Learning and Knowledge Discovery in Databases, 346-361, 2012.</p>
</dd>
</dl>
</div>
</div>
<div class="section" id="forests-of-randomized-trees">
<span id="forest"></span><h2>1.11.2. Forests of randomized trees<a class="headerlink" href="#forests-of-randomized-trees" title="Permalink to this headline">¶</a></h2>
<p>The <a class="reference internal" href="classes.html#module-sklearn.ensemble" title="sklearn.ensemble"><code class="xref py py-mod docutils literal notranslate"><span class="pre">sklearn.ensemble</span></code></a> module includes two averaging algorithms based
on randomized <a class="reference internal" href="tree.html#tree"><span class="std std-ref">decision trees</span></a>: the RandomForest algorithm
and the Extra-Trees method. Both algorithms are perturb-and-combine
techniques <a class="reference internal" href="#b1998" id="id5"><span>[B1998]</span></a> specifically designed for trees. This means a diverse
set of classifiers is created by introducing randomness in the classifier
construction.  The prediction of the ensemble is given as the averaged
prediction of the individual classifiers.</p>
<p>As other classifiers, forest classifiers have to be fitted with two
arrays: a sparse or dense array X of size <code class="docutils literal notranslate"><span class="pre">[n_samples,</span> <span class="pre">n_features]</span></code> holding the
training samples, and an array Y of size <code class="docutils literal notranslate"><span class="pre">[n_samples]</span></code> holding the
target values (class labels) for the training samples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">RandomForestClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span> <span class="o">=</span> <span class="p">[[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">Y</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">RandomForestClassifier</span><span class="p">(</span><span class="n">n_estimators</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">clf</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">Y</span><span class="p">)</span>
</pre></div>
</div>
<p>Like <a class="reference internal" href="tree.html#tree"><span class="std std-ref">decision trees</span></a>, forests of trees also extend
to <a class="reference internal" href="tree.html#tree-multioutput"><span class="std std-ref">multi-output problems</span></a>  (if Y is an array of size
<code class="docutils literal notranslate"><span class="pre">[n_samples,</span> <span class="pre">n_outputs]</span></code>).</p>
<div class="section" id="random-forests">
<h3>1.11.2.1. Random Forests<a class="headerlink" href="#random-forests" title="Permalink to this headline">¶</a></h3>
<p>In random forests (see <a class="reference internal" href="generated/sklearn.ensemble.RandomForestClassifier.html#sklearn.ensemble.RandomForestClassifier" title="sklearn.ensemble.RandomForestClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">RandomForestClassifier</span></code></a> and
<a class="reference internal" href="generated/sklearn.ensemble.RandomForestRegressor.html#sklearn.ensemble.RandomForestRegressor" title="sklearn.ensemble.RandomForestRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">RandomForestRegressor</span></code></a> classes), each tree in the ensemble is built
from a sample drawn with replacement (i.e., a bootstrap sample) from the
training set.</p>
<p>Furthermore, when splitting each node during the construction of a tree, the
best split is found either from all input features or a random subset of size
<code class="docutils literal notranslate"><span class="pre">max_features</span></code>. (See the <a class="reference internal" href="#random-forest-parameters"><span class="std std-ref">parameter tuning guidelines</span></a> for more details).</p>
<p>The purpose of these two sources of randomness is to decrease the variance of
the forest estimator. Indeed, individual decision trees typically exhibit high
variance and tend to overfit. The injected randomness in forests yield decision
trees with somewhat decoupled prediction errors. By taking an average of those
predictions, some errors can cancel out. Random forests achieve a reduced
variance by combining diverse trees, sometimes at the cost of a slight increase
in bias. In practice the variance reduction is often significant hence yielding
an overall better model.</p>
<p>In contrast to the original publication <a class="reference internal" href="#b2001" id="id6"><span>[B2001]</span></a>, the scikit-learn
implementation combines classifiers by averaging their probabilistic
prediction, instead of letting each classifier vote for a single class.</p>
</div>
<div class="section" id="extremely-randomized-trees">
<h3>1.11.2.2. Extremely Randomized Trees<a class="headerlink" href="#extremely-randomized-trees" title="Permalink to this headline">¶</a></h3>
<p>In extremely randomized trees (see <a class="reference internal" href="generated/sklearn.ensemble.ExtraTreesClassifier.html#sklearn.ensemble.ExtraTreesClassifier" title="sklearn.ensemble.ExtraTreesClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">ExtraTreesClassifier</span></code></a>
and <a class="reference internal" href="generated/sklearn.ensemble.ExtraTreesRegressor.html#sklearn.ensemble.ExtraTreesRegressor" title="sklearn.ensemble.ExtraTreesRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">ExtraTreesRegressor</span></code></a> classes), randomness goes one step
further in the way splits are computed. As in random forests, a random
subset of candidate features is used, but instead of looking for the
most discriminative thresholds, thresholds are drawn at random for each
candidate feature and the best of these randomly-generated thresholds is
picked as the splitting rule. This usually allows to reduce the variance
of the model a bit more, at the expense of a slightly greater increase
in bias:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">cross_val_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">make_blobs</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">RandomForestClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">ExtraTreesClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.tree</span> <span class="kn">import</span> <span class="n">DecisionTreeClassifier</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">make_blobs</span><span class="p">(</span><span class="n">n_samples</span><span class="o">=</span><span class="mi">10000</span><span class="p">,</span> <span class="n">n_features</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">centers</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">DecisionTreeClassifier</span><span class="p">(</span><span class="n">max_depth</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">min_samples_split</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scores</span> <span class="o">=</span> <span class="n">cross_val_score</span><span class="p">(</span><span class="n">clf</span><span class="p">,</span> <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scores</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
<span class="go">0.98...</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">RandomForestClassifier</span><span class="p">(</span><span class="n">n_estimators</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">max_depth</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">min_samples_split</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scores</span> <span class="o">=</span> <span class="n">cross_val_score</span><span class="p">(</span><span class="n">clf</span><span class="p">,</span> <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scores</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
<span class="go">0.999...</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">ExtraTreesClassifier</span><span class="p">(</span><span class="n">n_estimators</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">max_depth</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">min_samples_split</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scores</span> <span class="o">=</span> <span class="n">cross_val_score</span><span class="p">(</span><span class="n">clf</span><span class="p">,</span> <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scores</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mf">0.999</span>
<span class="go">True</span>
</pre></div>
</div>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/ensemble/plot_forest_iris.html"><img alt="modules/../auto_examples/ensemble/images/sphx_glr_plot_forest_iris_001.png" src="modules/../auto_examples/ensemble/images/sphx_glr_plot_forest_iris_001.png" /></a>
</div>
</div>
<div class="section" id="parameters">
<span id="random-forest-parameters"></span><h3>1.11.2.3. Parameters<a class="headerlink" href="#parameters" title="Permalink to this headline">¶</a></h3>
<p>The main parameters to adjust when using these methods is <code class="docutils literal notranslate"><span class="pre">n_estimators</span></code> and
<code class="docutils literal notranslate"><span class="pre">max_features</span></code>. The former is the number of trees in the forest. The larger
the better, but also the longer it will take to compute. In addition, note that
results will stop getting significantly better beyond a critical number of
trees. The latter is the size of the random subsets of features to consider
when splitting a node. The lower the greater the reduction of variance, but
also the greater the increase in bias. Empirical good default values are
<code class="docutils literal notranslate"><span class="pre">max_features=None</span></code> (always considering all features instead of a random
subset) for regression problems, and <code class="docutils literal notranslate"><span class="pre">max_features=&quot;sqrt&quot;</span></code> (using a random
subset of size <code class="docutils literal notranslate"><span class="pre">sqrt(n_features)</span></code>) for classification tasks (where
<code class="docutils literal notranslate"><span class="pre">n_features</span></code> is the number of features in the data). Good results are often
achieved when setting <code class="docutils literal notranslate"><span class="pre">max_depth=None</span></code> in combination with
<code class="docutils literal notranslate"><span class="pre">min_samples_split=2</span></code> (i.e., when fully developing the trees). Bear in mind
though that these values are usually not optimal, and might result in models
that consume a lot of RAM. The best parameter values should always be
cross-validated. In addition, note that in random forests, bootstrap samples
are used by default (<code class="docutils literal notranslate"><span class="pre">bootstrap=True</span></code>) while the default strategy for
extra-trees is to use the whole dataset (<code class="docutils literal notranslate"><span class="pre">bootstrap=False</span></code>). When using
bootstrap sampling the generalization accuracy can be estimated on the left out
or out-of-bag samples. This can be enabled by setting <code class="docutils literal notranslate"><span class="pre">oob_score=True</span></code>.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The size of the model with the default parameters is <span class="math notranslate nohighlight">\(O( M * N * log (N) )\)</span>,
where <span class="math notranslate nohighlight">\(M\)</span> is the number of trees and <span class="math notranslate nohighlight">\(N\)</span> is the number of samples.
In order to reduce the size of the model, you can change these parameters:
<code class="docutils literal notranslate"><span class="pre">min_samples_split</span></code>, <code class="docutils literal notranslate"><span class="pre">max_leaf_nodes</span></code>, <code class="docutils literal notranslate"><span class="pre">max_depth</span></code> and <code class="docutils literal notranslate"><span class="pre">min_samples_leaf</span></code>.</p>
</div>
</div>
<div class="section" id="parallelization">
<h3>1.11.2.4. Parallelization<a class="headerlink" href="#parallelization" title="Permalink to this headline">¶</a></h3>
<p>Finally, this module also features the parallel construction of the trees
and the parallel computation of the predictions through the <code class="docutils literal notranslate"><span class="pre">n_jobs</span></code>
parameter. If <code class="docutils literal notranslate"><span class="pre">n_jobs=k</span></code> then computations are partitioned into
<code class="docutils literal notranslate"><span class="pre">k</span></code> jobs, and run on <code class="docutils literal notranslate"><span class="pre">k</span></code> cores of the machine. If <code class="docutils literal notranslate"><span class="pre">n_jobs=-1</span></code>
then all cores available on the machine are used. Note that because of
inter-process communication overhead, the speedup might not be linear
(i.e., using <code class="docutils literal notranslate"><span class="pre">k</span></code> jobs will unfortunately not be <code class="docutils literal notranslate"><span class="pre">k</span></code> times as
fast). Significant speedup can still be achieved though when building
a large number of trees, or when building a single tree requires a fair
amount of time (e.g., on large datasets).</p>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_forest_iris.html#sphx-glr-auto-examples-ensemble-plot-forest-iris-py"><span class="std std-ref">Plot the decision surfaces of ensembles of trees on the iris dataset</span></a></p></li>
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_forest_importances_faces.html#sphx-glr-auto-examples-ensemble-plot-forest-importances-faces-py"><span class="std std-ref">Pixel importances with a parallel forest of trees</span></a></p></li>
<li><p><a class="reference internal" href="../auto_examples/plot_multioutput_face_completion.html#sphx-glr-auto-examples-plot-multioutput-face-completion-py"><span class="std std-ref">Face completion with a multi-output estimators</span></a></p></li>
</ul>
</div>
<div class="topic">
<p class="topic-title">References</p>
<dl class="citation">
<dt class="label" id="b2001"><span class="brackets"><a class="fn-backref" href="#id6">B2001</a></span></dt>
<dd><ol class="upperalpha simple" start="12">
<li><p>Breiman, “Random Forests”, Machine Learning, 45(1), 5-32, 2001.</p></li>
</ol>
</dd>
<dt class="label" id="b1998"><span class="brackets"><a class="fn-backref" href="#id5">B1998</a></span></dt>
<dd><ol class="upperalpha simple" start="12">
<li><p>Breiman, “Arcing Classifiers”, Annals of Statistics 1998.</p></li>
</ol>
</dd>
</dl>
<ul class="simple">
<li><p>P. Geurts, D. Ernst., and L. Wehenkel, “Extremely randomized
trees”, Machine Learning, 63(1), 3-42, 2006.</p></li>
</ul>
</div>
</div>
<div class="section" id="feature-importance-evaluation">
<span id="random-forest-feature-importance"></span><h3>1.11.2.5. Feature importance evaluation<a class="headerlink" href="#feature-importance-evaluation" title="Permalink to this headline">¶</a></h3>
<p>The relative rank (i.e. depth) of a feature used as a decision node in a
tree can be used to assess the relative importance of that feature with
respect to the predictability of the target variable. Features used at
the top of the tree contribute to the final prediction decision of a
larger fraction of the input samples. The <strong>expected fraction of the
samples</strong> they contribute to can thus be used as an estimate of the
<strong>relative importance of the features</strong>. In scikit-learn, the fraction of
samples a feature contributes to is combined with the decrease in impurity
from splitting them to create a normalized estimate of the predictive power
of that feature.</p>
<p>By <strong>averaging</strong> the estimates of predictive ability over several randomized
trees one can <strong>reduce the variance</strong> of such an estimate and use it
for feature selection. This is known as the mean decrease in impurity, or MDI.
Refer to <a class="reference internal" href="#l2014" id="id7"><span>[L2014]</span></a> for more information on MDI and feature importance
evaluation with Random Forests.</p>
<p>The following example shows a color-coded representation of the relative
importances of each individual pixel for a face recognition task using
a <a class="reference internal" href="generated/sklearn.ensemble.ExtraTreesClassifier.html#sklearn.ensemble.ExtraTreesClassifier" title="sklearn.ensemble.ExtraTreesClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">ExtraTreesClassifier</span></code></a> model.</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/ensemble/plot_forest_importances_faces.html"><img alt="modules/../auto_examples/ensemble/images/sphx_glr_plot_forest_importances_faces_001.png" src="modules/../auto_examples/ensemble/images/sphx_glr_plot_forest_importances_faces_001.png" /></a>
</div>
<p>In practice those estimates are stored as an attribute named
<code class="docutils literal notranslate"><span class="pre">feature_importances_</span></code> on the fitted model. This is an array with shape
<code class="docutils literal notranslate"><span class="pre">(n_features,)</span></code> whose values are positive and sum to 1.0. The higher
the value, the more important is the contribution of the matching feature
to the prediction function.</p>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_forest_importances_faces.html#sphx-glr-auto-examples-ensemble-plot-forest-importances-faces-py"><span class="std std-ref">Pixel importances with a parallel forest of trees</span></a></p></li>
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_forest_importances.html#sphx-glr-auto-examples-ensemble-plot-forest-importances-py"><span class="std std-ref">Feature importances with forests of trees</span></a></p></li>
</ul>
</div>
<div class="topic">
<p class="topic-title">References</p>
<dl class="citation">
<dt class="label" id="l2014"><span class="brackets"><a class="fn-backref" href="#id7">L2014</a></span></dt>
<dd><p>G. Louppe,
“Understanding Random Forests: From Theory to Practice”,
PhD Thesis, U. of Liege, 2014.</p>
</dd>
</dl>
</div>
</div>
<div class="section" id="totally-random-trees-embedding">
<span id="random-trees-embedding"></span><h3>1.11.2.6. Totally Random Trees Embedding<a class="headerlink" href="#totally-random-trees-embedding" title="Permalink to this headline">¶</a></h3>
<p><a class="reference internal" href="generated/sklearn.ensemble.RandomTreesEmbedding.html#sklearn.ensemble.RandomTreesEmbedding" title="sklearn.ensemble.RandomTreesEmbedding"><code class="xref py py-class docutils literal notranslate"><span class="pre">RandomTreesEmbedding</span></code></a> implements an unsupervised transformation of the
data.  Using a forest of completely random trees, <a class="reference internal" href="generated/sklearn.ensemble.RandomTreesEmbedding.html#sklearn.ensemble.RandomTreesEmbedding" title="sklearn.ensemble.RandomTreesEmbedding"><code class="xref py py-class docutils literal notranslate"><span class="pre">RandomTreesEmbedding</span></code></a>
encodes the data by the indices of the leaves a data point ends up in.  This
index is then encoded in a one-of-K manner, leading to a high dimensional,
sparse binary coding.
This coding can be computed very efficiently and can then be used as a basis
for other learning tasks.
The size and sparsity of the code can be influenced by choosing the number of
trees and the maximum depth per tree. For each tree in the ensemble, the coding
contains one entry of one. The size of the coding is at most <code class="docutils literal notranslate"><span class="pre">n_estimators</span> <span class="pre">*</span> <span class="pre">2</span>
<span class="pre">**</span> <span class="pre">max_depth</span></code>, the maximum number of leaves in the forest.</p>
<p>As neighboring data points are more likely to lie within the same leaf of a tree,
the transformation performs an implicit, non-parametric density estimation.</p>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_random_forest_embedding.html#sphx-glr-auto-examples-ensemble-plot-random-forest-embedding-py"><span class="std std-ref">Hashing feature transformation using Totally Random Trees</span></a></p></li>
<li><p><a class="reference internal" href="../auto_examples/manifold/plot_lle_digits.html#sphx-glr-auto-examples-manifold-plot-lle-digits-py"><span class="std std-ref">Manifold learning on handwritten digits: Locally Linear Embedding, Isomap…</span></a> compares non-linear
dimensionality reduction techniques on handwritten digits.</p></li>
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_feature_transformation.html#sphx-glr-auto-examples-ensemble-plot-feature-transformation-py"><span class="std std-ref">Feature transformations with ensembles of trees</span></a> compares
supervised and unsupervised tree based feature transformations.</p></li>
</ul>
</div>
<div class="admonition seealso">
<p class="admonition-title">See also</p>
<p><a class="reference internal" href="manifold.html#manifold"><span class="std std-ref">Manifold learning</span></a> techniques can also be useful to derive non-linear
representations of feature space, also these approaches focus also on
dimensionality reduction.</p>
</div>
</div>
</div>
<div class="section" id="adaboost">
<span id="id8"></span><h2>1.11.3. AdaBoost<a class="headerlink" href="#adaboost" title="Permalink to this headline">¶</a></h2>
<p>The module <a class="reference internal" href="classes.html#module-sklearn.ensemble" title="sklearn.ensemble"><code class="xref py py-mod docutils literal notranslate"><span class="pre">sklearn.ensemble</span></code></a> includes the popular boosting algorithm
AdaBoost, introduced in 1995 by Freund and Schapire <a class="reference internal" href="#fs1995" id="id9"><span>[FS1995]</span></a>.</p>
<p>The core principle of AdaBoost is to fit a sequence of weak learners (i.e.,
models that are only slightly better than random guessing, such as small
decision trees) on repeatedly modified versions of the data. The predictions
from all of them are then combined through a weighted majority vote (or sum) to
produce the final prediction. The data modifications at each so-called boosting
iteration consist of applying weights <span class="math notranslate nohighlight">\(w_1\)</span>, <span class="math notranslate nohighlight">\(w_2\)</span>, …, <span class="math notranslate nohighlight">\(w_N\)</span>
to each of the training samples. Initially, those weights are all set to
<span class="math notranslate nohighlight">\(w_i = 1/N\)</span>, so that the first step simply trains a weak learner on the
original data. For each successive iteration, the sample weights are
individually modified and the learning algorithm is reapplied to the reweighted
data. At a given step, those training examples that were incorrectly predicted
by the boosted model induced at the previous step have their weights increased,
whereas the weights are decreased for those that were predicted correctly. As
iterations proceed, examples that are difficult to predict receive
ever-increasing influence. Each subsequent weak learner is thereby forced to
concentrate on the examples that are missed by the previous ones in the sequence
<a class="reference internal" href="#htf" id="id10"><span>[HTF]</span></a>.</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/ensemble/plot_adaboost_hastie_10_2.html"><img alt="modules/../auto_examples/ensemble/images/sphx_glr_plot_adaboost_hastie_10_2_001.png" src="modules/../auto_examples/ensemble/images/sphx_glr_plot_adaboost_hastie_10_2_001.png" /></a>
</div>
<p>AdaBoost can be used both for classification and regression problems:</p>
<blockquote>
<div><ul class="simple">
<li><p>For multi-class classification, <a class="reference internal" href="generated/sklearn.ensemble.AdaBoostClassifier.html#sklearn.ensemble.AdaBoostClassifier" title="sklearn.ensemble.AdaBoostClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">AdaBoostClassifier</span></code></a> implements
AdaBoost-SAMME and AdaBoost-SAMME.R <a class="reference internal" href="#zzrh2009" id="id11"><span>[ZZRH2009]</span></a>.</p></li>
<li><p>For regression, <a class="reference internal" href="generated/sklearn.ensemble.AdaBoostRegressor.html#sklearn.ensemble.AdaBoostRegressor" title="sklearn.ensemble.AdaBoostRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">AdaBoostRegressor</span></code></a> implements AdaBoost.R2 <a class="reference internal" href="#d1997" id="id12"><span>[D1997]</span></a>.</p></li>
</ul>
</div></blockquote>
<div class="section" id="usage">
<h3>1.11.3.1. Usage<a class="headerlink" href="#usage" title="Permalink to this headline">¶</a></h3>
<p>The following example shows how to fit an AdaBoost classifier with 100 weak
learners:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">cross_val_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">load_iris</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">AdaBoostClassifier</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">load_iris</span><span class="p">(</span><span class="n">return_X_y</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">AdaBoostClassifier</span><span class="p">(</span><span class="n">n_estimators</span><span class="o">=</span><span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scores</span> <span class="o">=</span> <span class="n">cross_val_score</span><span class="p">(</span><span class="n">clf</span><span class="p">,</span> <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scores</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
<span class="go">0.9...</span>
</pre></div>
</div>
<p>The number of weak learners is controlled by the parameter <code class="docutils literal notranslate"><span class="pre">n_estimators</span></code>. The
<code class="docutils literal notranslate"><span class="pre">learning_rate</span></code> parameter controls the contribution of the weak learners in
the final combination. By default, weak learners are decision stumps. Different
weak learners can be specified through the <code class="docutils literal notranslate"><span class="pre">base_estimator</span></code> parameter.
The main parameters to tune to obtain good results are <code class="docutils literal notranslate"><span class="pre">n_estimators</span></code> and
the complexity of the base estimators (e.g., its depth <code class="docutils literal notranslate"><span class="pre">max_depth</span></code> or
minimum required number of samples to consider a split <code class="docutils literal notranslate"><span class="pre">min_samples_split</span></code>).</p>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_adaboost_hastie_10_2.html#sphx-glr-auto-examples-ensemble-plot-adaboost-hastie-10-2-py"><span class="std std-ref">Discrete versus Real AdaBoost</span></a> compares the
classification error of a decision stump, decision tree, and a boosted
decision stump using AdaBoost-SAMME and AdaBoost-SAMME.R.</p></li>
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_adaboost_multiclass.html#sphx-glr-auto-examples-ensemble-plot-adaboost-multiclass-py"><span class="std std-ref">Multi-class AdaBoosted Decision Trees</span></a> shows the performance
of AdaBoost-SAMME and AdaBoost-SAMME.R on a multi-class problem.</p></li>
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_adaboost_twoclass.html#sphx-glr-auto-examples-ensemble-plot-adaboost-twoclass-py"><span class="std std-ref">Two-class AdaBoost</span></a> shows the decision boundary
and decision function values for a non-linearly separable two-class problem
using AdaBoost-SAMME.</p></li>
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_adaboost_regression.html#sphx-glr-auto-examples-ensemble-plot-adaboost-regression-py"><span class="std std-ref">Decision Tree Regression with AdaBoost</span></a> demonstrates regression
with the AdaBoost.R2 algorithm.</p></li>
</ul>
</div>
<div class="topic">
<p class="topic-title">References</p>
<dl class="citation">
<dt class="label" id="fs1995"><span class="brackets"><a class="fn-backref" href="#id9">FS1995</a></span></dt>
<dd><p>Y. Freund, and R. Schapire, “A Decision-Theoretic Generalization of
On-Line Learning and an Application to Boosting”, 1997.</p>
</dd>
<dt class="label" id="zzrh2009"><span class="brackets"><a class="fn-backref" href="#id11">ZZRH2009</a></span></dt>
<dd><p>J. Zhu, H. Zou, S. Rosset, T. Hastie. “Multi-class AdaBoost”,
2009.</p>
</dd>
<dt class="label" id="d1997"><span class="brackets"><a class="fn-backref" href="#id12">D1997</a></span></dt>
<dd><ol class="upperalpha simple" start="8">
<li><p>Drucker. “Improving Regressors using Boosting Techniques”, 1997.</p></li>
</ol>
</dd>
<dt class="label" id="htf"><span class="brackets">HTF</span><span class="fn-backref">(<a href="#id10">1</a>,<a href="#id18">2</a>,<a href="#id31">3</a>)</span></dt>
<dd><p>T. Hastie, R. Tibshirani and J. Friedman, “Elements of
Statistical Learning Ed. 2”, Springer, 2009.</p>
</dd>
</dl>
</div>
</div>
</div>
<div class="section" id="gradient-tree-boosting">
<span id="gradient-boosting"></span><h2>1.11.4. Gradient Tree Boosting<a class="headerlink" href="#gradient-tree-boosting" title="Permalink to this headline">¶</a></h2>
<p><a class="reference external" href="https://en.wikipedia.org/wiki/Gradient_boosting">Gradient Tree Boosting</a>
or Gradient Boosted Decision Trees (GBDT) is a generalization
of boosting to arbitrary
differentiable loss functions. GBDT is an accurate and effective
off-the-shelf procedure that can be used for both regression and
classification problems in a
variety of areas including Web search ranking and ecology.</p>
<p>The module <a class="reference internal" href="classes.html#module-sklearn.ensemble" title="sklearn.ensemble"><code class="xref py py-mod docutils literal notranslate"><span class="pre">sklearn.ensemble</span></code></a> provides methods
for both classification and regression via gradient boosted decision
trees.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Scikit-learn 0.21 introduces two new experimental implementations of
gradient boosting trees, namely <a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingClassifier.html#sklearn.ensemble.HistGradientBoostingClassifier" title="sklearn.ensemble.HistGradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingClassifier</span></code></a>
and <a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingRegressor.html#sklearn.ensemble.HistGradientBoostingRegressor" title="sklearn.ensemble.HistGradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingRegressor</span></code></a>, inspired by
<a class="reference external" href="https://github.com/Microsoft/LightGBM">LightGBM</a> (See <a class="reference internal" href="#lightgbm" id="id14"><span>[LightGBM]</span></a>).</p>
<p>These histogram-based estimators can be <strong>orders of magnitude faster</strong>
than <a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingClassifier.html#sklearn.ensemble.GradientBoostingClassifier" title="sklearn.ensemble.GradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingClassifier</span></code></a> and
<a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingRegressor.html#sklearn.ensemble.GradientBoostingRegressor" title="sklearn.ensemble.GradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingRegressor</span></code></a> when the number of samples is larger
than tens of thousands of samples.</p>
<p>They also have built-in support for missing values, which avoids the need
for an imputer.</p>
<p>These estimators are described in more detail below in
<a class="reference internal" href="#histogram-based-gradient-boosting"><span class="std std-ref">Histogram-Based Gradient Boosting</span></a>.</p>
<p>The following guide focuses on <a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingClassifier.html#sklearn.ensemble.GradientBoostingClassifier" title="sklearn.ensemble.GradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingClassifier</span></code></a> and
<a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingRegressor.html#sklearn.ensemble.GradientBoostingRegressor" title="sklearn.ensemble.GradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingRegressor</span></code></a>, which might be preferred for small
sample sizes since binning may lead to split points that are too approximate
in this setting.</p>
</div>
<div class="section" id="classification">
<h3>1.11.4.1. Classification<a class="headerlink" href="#classification" title="Permalink to this headline">¶</a></h3>
<p><a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingClassifier.html#sklearn.ensemble.GradientBoostingClassifier" title="sklearn.ensemble.GradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingClassifier</span></code></a> supports both binary and multi-class
classification.
The following example shows how to fit a gradient boosting classifier
with 100 decision stumps as weak learners:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">make_hastie_10_2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">GradientBoostingClassifier</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">make_hastie_10_2</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span> <span class="o">=</span> <span class="n">X</span><span class="p">[:</span><span class="mi">2000</span><span class="p">],</span> <span class="n">X</span><span class="p">[</span><span class="mi">2000</span><span class="p">:]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">y</span><span class="p">[:</span><span class="mi">2000</span><span class="p">],</span> <span class="n">y</span><span class="p">[</span><span class="mi">2000</span><span class="p">:]</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">GradientBoostingClassifier</span><span class="p">(</span><span class="n">n_estimators</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">learning_rate</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">max_depth</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span>
<span class="go">0.913...</span>
</pre></div>
</div>
<p>The number of weak learners (i.e. regression trees) is controlled by the parameter <code class="docutils literal notranslate"><span class="pre">n_estimators</span></code>; <a class="reference internal" href="#gradient-boosting-tree-size"><span class="std std-ref">The size of each tree</span></a> can be controlled either by setting the tree depth via <code class="docutils literal notranslate"><span class="pre">max_depth</span></code> or by setting the number of leaf nodes via <code class="docutils literal notranslate"><span class="pre">max_leaf_nodes</span></code>. The <code class="docutils literal notranslate"><span class="pre">learning_rate</span></code> is a hyper-parameter in the range (0.0, 1.0] that controls overfitting via <a class="reference internal" href="#gradient-boosting-shrinkage"><span class="std std-ref">shrinkage</span></a> .</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Classification with more than 2 classes requires the induction
of <code class="docutils literal notranslate"><span class="pre">n_classes</span></code> regression trees at each iteration,
thus, the total number of induced trees equals
<code class="docutils literal notranslate"><span class="pre">n_classes</span> <span class="pre">*</span> <span class="pre">n_estimators</span></code>. For datasets with a large number
of classes we strongly recommend to use
<a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingClassifier.html#sklearn.ensemble.HistGradientBoostingClassifier" title="sklearn.ensemble.HistGradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingClassifier</span></code></a> as an alternative to
<a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingClassifier.html#sklearn.ensemble.GradientBoostingClassifier" title="sklearn.ensemble.GradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingClassifier</span></code></a> .</p>
</div>
</div>
<div class="section" id="regression">
<h3>1.11.4.2. Regression<a class="headerlink" href="#regression" title="Permalink to this headline">¶</a></h3>
<p><a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingRegressor.html#sklearn.ensemble.GradientBoostingRegressor" title="sklearn.ensemble.GradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingRegressor</span></code></a> supports a number of
<a class="reference internal" href="#gradient-boosting-loss"><span class="std std-ref">different loss functions</span></a>
for regression which can be specified via the argument
<code class="docutils literal notranslate"><span class="pre">loss</span></code>; the default loss function for regression is least squares (<code class="docutils literal notranslate"><span class="pre">'ls'</span></code>).</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">mean_squared_error</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">make_friedman1</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">GradientBoostingRegressor</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">make_friedman1</span><span class="p">(</span><span class="n">n_samples</span><span class="o">=</span><span class="mi">1200</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">noise</span><span class="o">=</span><span class="mf">1.0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span> <span class="o">=</span> <span class="n">X</span><span class="p">[:</span><span class="mi">200</span><span class="p">],</span> <span class="n">X</span><span class="p">[</span><span class="mi">200</span><span class="p">:]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">y</span><span class="p">[:</span><span class="mi">200</span><span class="p">],</span> <span class="n">y</span><span class="p">[</span><span class="mi">200</span><span class="p">:]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">est</span> <span class="o">=</span> <span class="n">GradientBoostingRegressor</span><span class="p">(</span><span class="n">n_estimators</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">learning_rate</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">max_depth</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">loss</span><span class="o">=</span><span class="s1">&#39;ls&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_squared_error</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span> <span class="n">est</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_test</span><span class="p">))</span>
<span class="go">5.00...</span>
</pre></div>
</div>
<p>The figure below shows the results of applying <a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingRegressor.html#sklearn.ensemble.GradientBoostingRegressor" title="sklearn.ensemble.GradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingRegressor</span></code></a>
with least squares loss and 500 base learners to the Boston house price dataset
(<a class="reference internal" href="generated/sklearn.datasets.load_boston.html#sklearn.datasets.load_boston" title="sklearn.datasets.load_boston"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.load_boston</span></code></a>).
The plot on the left shows the train and test error at each iteration.
The train error at each iteration is stored in the
<code class="xref py py-attr docutils literal notranslate"><span class="pre">train_score_</span></code> attribute
of the gradient boosting model. The test error at each iterations can be obtained
via the <a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingRegressor.html#sklearn.ensemble.GradientBoostingRegressor.staged_predict" title="sklearn.ensemble.GradientBoostingRegressor.staged_predict"><code class="xref py py-meth docutils literal notranslate"><span class="pre">staged_predict</span></code></a> method which returns a
generator that yields the predictions at each stage. Plots like these can be used
to determine the optimal number of trees (i.e. <code class="docutils literal notranslate"><span class="pre">n_estimators</span></code>) by early stopping.
The plot on the right shows the feature importances which can be obtained via
the <code class="docutils literal notranslate"><span class="pre">feature_importances_</span></code> property.</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/ensemble/plot_gradient_boosting_regression.html"><img alt="modules/../auto_examples/ensemble/images/sphx_glr_plot_gradient_boosting_regression_001.png" src="modules/../auto_examples/ensemble/images/sphx_glr_plot_gradient_boosting_regression_001.png" /></a>
</div>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_gradient_boosting_regression.html#sphx-glr-auto-examples-ensemble-plot-gradient-boosting-regression-py"><span class="std std-ref">Gradient Boosting regression</span></a></p></li>
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_gradient_boosting_oob.html#sphx-glr-auto-examples-ensemble-plot-gradient-boosting-oob-py"><span class="std std-ref">Gradient Boosting Out-of-Bag estimates</span></a></p></li>
</ul>
</div>
</div>
<div class="section" id="fitting-additional-weak-learners">
<span id="gradient-boosting-warm-start"></span><h3>1.11.4.3. Fitting additional weak-learners<a class="headerlink" href="#fitting-additional-weak-learners" title="Permalink to this headline">¶</a></h3>
<p>Both <a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingRegressor.html#sklearn.ensemble.GradientBoostingRegressor" title="sklearn.ensemble.GradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingRegressor</span></code></a> and <a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingClassifier.html#sklearn.ensemble.GradientBoostingClassifier" title="sklearn.ensemble.GradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingClassifier</span></code></a>
support <code class="docutils literal notranslate"><span class="pre">warm_start=True</span></code> which allows you to add more estimators to an already
fitted model.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">_</span> <span class="o">=</span> <span class="n">est</span><span class="o">.</span><span class="n">set_params</span><span class="p">(</span><span class="n">n_estimators</span><span class="o">=</span><span class="mi">200</span><span class="p">,</span> <span class="n">warm_start</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>  <span class="c1"># set warm_start and new nr of trees</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">_</span> <span class="o">=</span> <span class="n">est</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span> <span class="c1"># fit additional 100 trees to est</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_squared_error</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span> <span class="n">est</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_test</span><span class="p">))</span>
<span class="go">3.84...</span>
</pre></div>
</div>
</div>
<div class="section" id="controlling-the-tree-size">
<span id="gradient-boosting-tree-size"></span><h3>1.11.4.4. Controlling the tree size<a class="headerlink" href="#controlling-the-tree-size" title="Permalink to this headline">¶</a></h3>
<p>The size of the regression tree base learners defines the level of variable
interactions that can be captured by the gradient boosting model. In general,
a tree of depth <code class="docutils literal notranslate"><span class="pre">h</span></code> can capture interactions of order <code class="docutils literal notranslate"><span class="pre">h</span></code> .
There are two ways in which the size of the individual regression trees can
be controlled.</p>
<p>If you specify <code class="docutils literal notranslate"><span class="pre">max_depth=h</span></code> then complete binary trees
of depth <code class="docutils literal notranslate"><span class="pre">h</span></code> will be grown. Such trees will have (at most) <code class="docutils literal notranslate"><span class="pre">2**h</span></code> leaf nodes
and <code class="docutils literal notranslate"><span class="pre">2**h</span> <span class="pre">-</span> <span class="pre">1</span></code> split nodes.</p>
<p>Alternatively, you can control the tree size by specifying the number of
leaf nodes via the parameter <code class="docutils literal notranslate"><span class="pre">max_leaf_nodes</span></code>. In this case,
trees will be grown using best-first search where nodes with the highest improvement
in impurity will be expanded first.
A tree with <code class="docutils literal notranslate"><span class="pre">max_leaf_nodes=k</span></code> has <code class="docutils literal notranslate"><span class="pre">k</span> <span class="pre">-</span> <span class="pre">1</span></code> split nodes and thus can
model interactions of up to order <code class="docutils literal notranslate"><span class="pre">max_leaf_nodes</span> <span class="pre">-</span> <span class="pre">1</span></code> .</p>
<p>We found that <code class="docutils literal notranslate"><span class="pre">max_leaf_nodes=k</span></code> gives comparable results to <code class="docutils literal notranslate"><span class="pre">max_depth=k-1</span></code>
but is significantly faster to train at the expense of a slightly higher
training error.
The parameter <code class="docutils literal notranslate"><span class="pre">max_leaf_nodes</span></code> corresponds to the variable <code class="docutils literal notranslate"><span class="pre">J</span></code> in the
chapter on gradient boosting in <a class="reference internal" href="model_evaluation.html#f2001" id="id15"><span>[F2001]</span></a> and is related to the parameter
<code class="docutils literal notranslate"><span class="pre">interaction.depth</span></code> in R’s gbm package where <code class="docutils literal notranslate"><span class="pre">max_leaf_nodes</span> <span class="pre">==</span> <span class="pre">interaction.depth</span> <span class="pre">+</span> <span class="pre">1</span></code> .</p>
</div>
<div class="section" id="mathematical-formulation">
<h3>1.11.4.5. Mathematical formulation<a class="headerlink" href="#mathematical-formulation" title="Permalink to this headline">¶</a></h3>
<p>GBRT considers additive models of the following form:</p>
<blockquote>
<div><div class="math notranslate nohighlight">
\[F(x) = \sum_{m=1}^{M} \gamma_m h_m(x)\]</div>
</div></blockquote>
<p>where <span class="math notranslate nohighlight">\(h_m(x)\)</span> are the basis functions which are usually called
<em>weak learners</em> in the context of boosting. Gradient Tree Boosting
uses <a class="reference internal" href="tree.html#tree"><span class="std std-ref">decision trees</span></a> of fixed size as weak
learners. Decision trees have a number of abilities that make them
valuable for boosting, namely the ability to handle data of mixed type
and the ability to model complex functions.</p>
<p>Similar to other boosting algorithms, GBRT builds the additive model in
a greedy fashion:</p>
<blockquote>
<div><div class="math notranslate nohighlight">
\[F_m(x) = F_{m-1}(x) + \gamma_m h_m(x),\]</div>
</div></blockquote>
<p>where the newly added tree <span class="math notranslate nohighlight">\(h_m\)</span> tries to minimize the loss <span class="math notranslate nohighlight">\(L\)</span>,
given the previous ensemble <span class="math notranslate nohighlight">\(F_{m-1}\)</span>:</p>
<blockquote>
<div><div class="math notranslate nohighlight">
\[h_m =  \arg\min_{h} \sum_{i=1}^{n} L(y_i,
F_{m-1}(x_i) + h(x_i)).\]</div>
</div></blockquote>
<p>The initial model <span class="math notranslate nohighlight">\(F_{0}\)</span> is problem specific, for least-squares
regression one usually chooses the mean of the target values.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The initial model can also be specified via the <code class="docutils literal notranslate"><span class="pre">init</span></code>
argument. The passed object has to implement <code class="docutils literal notranslate"><span class="pre">fit</span></code> and <code class="docutils literal notranslate"><span class="pre">predict</span></code>.</p>
</div>
<p>Gradient Boosting attempts to solve this minimization problem
numerically via steepest descent: The steepest descent direction is
the negative gradient of the loss function evaluated at the current
model <span class="math notranslate nohighlight">\(F_{m-1}\)</span> which can be calculated for any differentiable
loss function:</p>
<blockquote>
<div><div class="math notranslate nohighlight">
\[F_m(x) = F_{m-1}(x) - \gamma_m \sum_{i=1}^{n} \nabla_F L(y_i,
F_{m-1}(x_i))\]</div>
</div></blockquote>
<p>Where the step length <span class="math notranslate nohighlight">\(\gamma_m\)</span> is chosen using line search:</p>
<blockquote>
<div><div class="math notranslate nohighlight">
\[\gamma_m = \arg\min_{\gamma} \sum_{i=1}^{n} L(y_i, F_{m-1}(x_i)
- \gamma \frac{\partial L(y_i, F_{m-1}(x_i))}{\partial F_{m-1}(x_i)})\]</div>
</div></blockquote>
<p>The algorithms for regression and classification
only differ in the concrete loss function used.</p>
<div class="section" id="loss-functions">
<span id="gradient-boosting-loss"></span><h4>1.11.4.5.1. Loss Functions<a class="headerlink" href="#loss-functions" title="Permalink to this headline">¶</a></h4>
<p>The following loss functions are supported and can be specified using
the parameter <code class="docutils literal notranslate"><span class="pre">loss</span></code>:</p>
<blockquote>
<div><ul class="simple">
<li><p>Regression</p>
<ul>
<li><p>Least squares (<code class="docutils literal notranslate"><span class="pre">'ls'</span></code>): The natural choice for regression due
to its superior computational properties. The initial model is
given by the mean of the target values.</p></li>
<li><p>Least absolute deviation (<code class="docutils literal notranslate"><span class="pre">'lad'</span></code>): A robust loss function for
regression. The initial model is given by the median of the
target values.</p></li>
<li><p>Huber (<code class="docutils literal notranslate"><span class="pre">'huber'</span></code>): Another robust loss function that combines
least squares and least absolute deviation; use <code class="docutils literal notranslate"><span class="pre">alpha</span></code> to
control the sensitivity with regards to outliers (see <a class="reference internal" href="model_evaluation.html#f2001" id="id16"><span>[F2001]</span></a> for
more details).</p></li>
<li><p>Quantile (<code class="docutils literal notranslate"><span class="pre">'quantile'</span></code>): A loss function for quantile regression.
Use <code class="docutils literal notranslate"><span class="pre">0</span> <span class="pre">&lt;</span> <span class="pre">alpha</span> <span class="pre">&lt;</span> <span class="pre">1</span></code> to specify the quantile. This loss function
can be used to create prediction intervals
(see <a class="reference internal" href="../auto_examples/ensemble/plot_gradient_boosting_quantile.html#sphx-glr-auto-examples-ensemble-plot-gradient-boosting-quantile-py"><span class="std std-ref">Prediction Intervals for Gradient Boosting Regression</span></a>).</p></li>
</ul>
</li>
<li><p>Classification</p>
<ul>
<li><p>Binomial deviance (<code class="docutils literal notranslate"><span class="pre">'deviance'</span></code>): The negative binomial
log-likelihood loss function for binary classification (provides
probability estimates).  The initial model is given by the
log odds-ratio.</p></li>
<li><p>Multinomial deviance (<code class="docutils literal notranslate"><span class="pre">'deviance'</span></code>): The negative multinomial
log-likelihood loss function for multi-class classification with
<code class="docutils literal notranslate"><span class="pre">n_classes</span></code> mutually exclusive classes. It provides
probability estimates.  The initial model is given by the
prior probability of each class. At each iteration <code class="docutils literal notranslate"><span class="pre">n_classes</span></code>
regression trees have to be constructed which makes GBRT rather
inefficient for data sets with a large number of classes.</p></li>
<li><p>Exponential loss (<code class="docutils literal notranslate"><span class="pre">'exponential'</span></code>): The same loss function
as <a class="reference internal" href="generated/sklearn.ensemble.AdaBoostClassifier.html#sklearn.ensemble.AdaBoostClassifier" title="sklearn.ensemble.AdaBoostClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">AdaBoostClassifier</span></code></a>. Less robust to mislabeled
examples than <code class="docutils literal notranslate"><span class="pre">'deviance'</span></code>; can only be used for binary
classification.</p></li>
</ul>
</li>
</ul>
</div></blockquote>
</div>
</div>
<div class="section" id="regularization">
<h3>1.11.4.6. Regularization<a class="headerlink" href="#regularization" title="Permalink to this headline">¶</a></h3>
<div class="section" id="shrinkage">
<span id="gradient-boosting-shrinkage"></span><h4>1.11.4.6.1. Shrinkage<a class="headerlink" href="#shrinkage" title="Permalink to this headline">¶</a></h4>
<p><a class="reference internal" href="model_evaluation.html#f2001" id="id17"><span>[F2001]</span></a> proposed a simple regularization strategy that scales
the contribution of each weak learner by a factor <span class="math notranslate nohighlight">\(\nu\)</span>:</p>
<div class="math notranslate nohighlight">
\[F_m(x) = F_{m-1}(x) + \nu \gamma_m h_m(x)\]</div>
<p>The parameter <span class="math notranslate nohighlight">\(\nu\)</span> is also called the <strong>learning rate</strong> because
it scales the step length the gradient descent procedure; it can
be set via the <code class="docutils literal notranslate"><span class="pre">learning_rate</span></code> parameter.</p>
<p>The parameter <code class="docutils literal notranslate"><span class="pre">learning_rate</span></code> strongly interacts with the parameter
<code class="docutils literal notranslate"><span class="pre">n_estimators</span></code>, the number of weak learners to fit. Smaller values
of <code class="docutils literal notranslate"><span class="pre">learning_rate</span></code> require larger numbers of weak learners to maintain
a constant training error. Empirical evidence suggests that small
values of <code class="docutils literal notranslate"><span class="pre">learning_rate</span></code> favor better test error. <a class="reference internal" href="#htf" id="id18"><span>[HTF]</span></a>
recommend to set the learning rate to a small constant
(e.g. <code class="docutils literal notranslate"><span class="pre">learning_rate</span> <span class="pre">&lt;=</span> <span class="pre">0.1</span></code>) and choose <code class="docutils literal notranslate"><span class="pre">n_estimators</span></code> by early
stopping. For a more detailed discussion of the interaction between
<code class="docutils literal notranslate"><span class="pre">learning_rate</span></code> and <code class="docutils literal notranslate"><span class="pre">n_estimators</span></code> see <a class="reference internal" href="#r2007" id="id19"><span>[R2007]</span></a>.</p>
</div>
<div class="section" id="subsampling">
<h4>1.11.4.6.2. Subsampling<a class="headerlink" href="#subsampling" title="Permalink to this headline">¶</a></h4>
<p><a class="reference internal" href="#f1999" id="id20"><span>[F1999]</span></a> proposed stochastic gradient boosting, which combines gradient
boosting with bootstrap averaging (bagging). At each iteration
the base classifier is trained on a fraction <code class="docutils literal notranslate"><span class="pre">subsample</span></code> of
the available training data. The subsample is drawn without replacement.
A typical value of <code class="docutils literal notranslate"><span class="pre">subsample</span></code> is 0.5.</p>
<p>The figure below illustrates the effect of shrinkage and subsampling
on the goodness-of-fit of the model. We can clearly see that shrinkage
outperforms no-shrinkage. Subsampling with shrinkage can further increase
the accuracy of the model. Subsampling without shrinkage, on the other hand,
does poorly.</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/ensemble/plot_gradient_boosting_regularization.html"><img alt="modules/../auto_examples/ensemble/images/sphx_glr_plot_gradient_boosting_regularization_001.png" src="modules/../auto_examples/ensemble/images/sphx_glr_plot_gradient_boosting_regularization_001.png" /></a>
</div>
<p>Another strategy to reduce the variance is by subsampling the features
analogous to the random splits in <a class="reference internal" href="generated/sklearn.ensemble.RandomForestClassifier.html#sklearn.ensemble.RandomForestClassifier" title="sklearn.ensemble.RandomForestClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">RandomForestClassifier</span></code></a> .
The number of subsampled features can be controlled via the <code class="docutils literal notranslate"><span class="pre">max_features</span></code>
parameter.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Using a small <code class="docutils literal notranslate"><span class="pre">max_features</span></code> value can significantly decrease the runtime.</p>
</div>
<p>Stochastic gradient boosting allows to compute out-of-bag estimates of the
test deviance by computing the improvement in deviance on the examples that are
not included in the bootstrap sample (i.e. the out-of-bag examples).
The improvements are stored in the attribute
<code class="xref py py-attr docutils literal notranslate"><span class="pre">oob_improvement_</span></code>. <code class="docutils literal notranslate"><span class="pre">oob_improvement_[i]</span></code> holds
the improvement in terms of the loss on the OOB samples if you add the i-th stage
to the current predictions.
Out-of-bag estimates can be used for model selection, for example to determine
the optimal number of iterations. OOB estimates are usually very pessimistic thus
we recommend to use cross-validation instead and only use OOB if cross-validation
is too time consuming.</p>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_gradient_boosting_regularization.html#sphx-glr-auto-examples-ensemble-plot-gradient-boosting-regularization-py"><span class="std std-ref">Gradient Boosting regularization</span></a></p></li>
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_gradient_boosting_oob.html#sphx-glr-auto-examples-ensemble-plot-gradient-boosting-oob-py"><span class="std std-ref">Gradient Boosting Out-of-Bag estimates</span></a></p></li>
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_ensemble_oob.html#sphx-glr-auto-examples-ensemble-plot-ensemble-oob-py"><span class="std std-ref">OOB Errors for Random Forests</span></a></p></li>
</ul>
</div>
</div>
</div>
<div class="section" id="interpretation">
<h3>1.11.4.7. Interpretation<a class="headerlink" href="#interpretation" title="Permalink to this headline">¶</a></h3>
<p>Individual decision trees can be interpreted easily by simply
visualizing the tree structure. Gradient boosting models, however,
comprise hundreds of regression trees thus they cannot be easily
interpreted by visual inspection of the individual trees. Fortunately,
a number of techniques have been proposed to summarize and interpret
gradient boosting models.</p>
<div class="section" id="feature-importance">
<h4>1.11.4.7.1. Feature importance<a class="headerlink" href="#feature-importance" title="Permalink to this headline">¶</a></h4>
<p>Often features do not contribute equally to predict the target
response; in many situations the majority of the features are in fact
irrelevant.
When interpreting a model, the first question usually is: what are
those important features and how do they contributing in predicting
the target response?</p>
<p>Individual decision trees intrinsically perform feature selection by selecting
appropriate split points. This information can be used to measure the
importance of each feature; the basic idea is: the more often a
feature is used in the split points of a tree the more important that
feature is. This notion of importance can be extended to decision tree
ensembles by simply averaging the feature importance of each tree (see
<a class="reference internal" href="#random-forest-feature-importance"><span class="std std-ref">Feature importance evaluation</span></a> for more details).</p>
<p>The feature importance scores of a fit gradient boosting model can be
accessed via the <code class="docutils literal notranslate"><span class="pre">feature_importances_</span></code> property:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">make_hastie_10_2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">GradientBoostingClassifier</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">make_hastie_10_2</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">GradientBoostingClassifier</span><span class="p">(</span><span class="n">n_estimators</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">learning_rate</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">max_depth</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span><span class="o">.</span><span class="n">feature_importances_</span>
<span class="go">array([0.10..., 0.10..., 0.11..., ...</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_gradient_boosting_regression.html#sphx-glr-auto-examples-ensemble-plot-gradient-boosting-regression-py"><span class="std std-ref">Gradient Boosting regression</span></a></p></li>
</ul>
</div>
</div>
</div>
</div>
<div class="section" id="histogram-based-gradient-boosting">
<span id="id21"></span><h2>1.11.5. Histogram-Based Gradient Boosting<a class="headerlink" href="#histogram-based-gradient-boosting" title="Permalink to this headline">¶</a></h2>
<p>Scikit-learn 0.21 introduces two new experimental implementations of
gradient boosting trees, namely <a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingClassifier.html#sklearn.ensemble.HistGradientBoostingClassifier" title="sklearn.ensemble.HistGradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingClassifier</span></code></a>
and <a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingRegressor.html#sklearn.ensemble.HistGradientBoostingRegressor" title="sklearn.ensemble.HistGradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingRegressor</span></code></a>, inspired by
<a class="reference external" href="https://github.com/Microsoft/LightGBM">LightGBM</a> (See <a class="reference internal" href="#lightgbm" id="id22"><span>[LightGBM]</span></a>).</p>
<p>These histogram-based estimators can be <strong>orders of magnitude faster</strong>
than <a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingClassifier.html#sklearn.ensemble.GradientBoostingClassifier" title="sklearn.ensemble.GradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingClassifier</span></code></a> and
<a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingRegressor.html#sklearn.ensemble.GradientBoostingRegressor" title="sklearn.ensemble.GradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingRegressor</span></code></a> when the number of samples is larger
than tens of thousands of samples.</p>
<p>They also have built-in support for missing values, which avoids the need
for an imputer.</p>
<p>These fast estimators first bin the input samples <code class="docutils literal notranslate"><span class="pre">X</span></code> into
integer-valued bins (typically 256 bins) which tremendously reduces the
number of splitting points to consider, and allows the algorithm to
leverage integer-based data structures (histograms) instead of relying on
sorted continuous values when building the trees. The API of these
estimators is slightly different, and some of the features from
<a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingClassifier.html#sklearn.ensemble.GradientBoostingClassifier" title="sklearn.ensemble.GradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingClassifier</span></code></a> and <a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingRegressor.html#sklearn.ensemble.GradientBoostingRegressor" title="sklearn.ensemble.GradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingRegressor</span></code></a>
are not yet supported: in particular sample weights, and some loss
functions.</p>
<p>These estimators are still <strong>experimental</strong>: their predictions
and their API might change without any deprecation cycle. To use them, you
need to explicitly import <code class="docutils literal notranslate"><span class="pre">enable_hist_gradient_boosting</span></code>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># explicitly require this experimental feature</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.experimental</span> <span class="kn">import</span> <span class="n">enable_hist_gradient_boosting</span>  <span class="c1"># noqa</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># now you can import normally from ensemble</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">HistGradientBoostingClassifier</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p><a class="reference internal" href="../auto_examples/inspection/plot_partial_dependence.html#sphx-glr-auto-examples-inspection-plot-partial-dependence-py"><span class="std std-ref">Partial Dependence Plots</span></a></p></li>
</ul>
</div>
<div class="section" id="id23">
<h3>1.11.5.1. Usage<a class="headerlink" href="#id23" title="Permalink to this headline">¶</a></h3>
<p>Most of the parameters are unchanged from
<a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingClassifier.html#sklearn.ensemble.GradientBoostingClassifier" title="sklearn.ensemble.GradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingClassifier</span></code></a> and <a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingRegressor.html#sklearn.ensemble.GradientBoostingRegressor" title="sklearn.ensemble.GradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingRegressor</span></code></a>.
One exception is the <code class="docutils literal notranslate"><span class="pre">max_iter</span></code> parameter that replaces <code class="docutils literal notranslate"><span class="pre">n_estimators</span></code>, and
controls the number of iterations of the boosting process:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.experimental</span> <span class="kn">import</span> <span class="n">enable_hist_gradient_boosting</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">HistGradientBoostingClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">make_hastie_10_2</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">make_hastie_10_2</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span> <span class="o">=</span> <span class="n">X</span><span class="p">[:</span><span class="mi">2000</span><span class="p">],</span> <span class="n">X</span><span class="p">[</span><span class="mi">2000</span><span class="p">:]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">y</span><span class="p">[:</span><span class="mi">2000</span><span class="p">],</span> <span class="n">y</span><span class="p">[</span><span class="mi">2000</span><span class="p">:]</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">HistGradientBoostingClassifier</span><span class="p">(</span><span class="n">max_iter</span><span class="o">=</span><span class="mi">100</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span>
<span class="go">0.8965</span>
</pre></div>
</div>
<p>Available losses for regression are ‘least_squares’ and
‘least_absolute_deviation’, which is less sensitive to outliers. For
classification, ‘binary_crossentropy’ is used for binary classification and
‘categorical_crossentropy’ is used for multiclass classification. By default
the loss is ‘auto’ and will select the appropriate loss depending on
<a class="reference internal" href="../glossary.html#term-177"><span class="xref std std-term">y</span></a> passed to <a class="reference internal" href="../glossary.html#term-fit"><span class="xref std std-term">fit</span></a>.</p>
<p>The size of the trees can be controlled through the <code class="docutils literal notranslate"><span class="pre">max_leaf_nodes</span></code>,
<code class="docutils literal notranslate"><span class="pre">max_depth</span></code>, and <code class="docutils literal notranslate"><span class="pre">min_samples_leaf</span></code> parameters.</p>
<p>The number of bins used to bin the data is controlled with the <code class="docutils literal notranslate"><span class="pre">max_bins</span></code>
parameter. Using less bins acts as a form of regularization. It is
generally recommended to use as many bins as possible, which is the default.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">l2_regularization</span></code> parameter is a regularizer on the loss function and
corresponds to <span class="math notranslate nohighlight">\(\lambda\)</span> in equation (2) of <a class="reference internal" href="#xgboost" id="id24"><span>[XGBoost]</span></a>.</p>
<p>The early-stopping behaviour is controlled via the <code class="docutils literal notranslate"><span class="pre">scoring</span></code>,
<code class="docutils literal notranslate"><span class="pre">validation_fraction</span></code>, <code class="docutils literal notranslate"><span class="pre">n_iter_no_change</span></code>, and <code class="docutils literal notranslate"><span class="pre">tol</span></code> parameters. It is
possible to early-stop using an arbitrary <a class="reference internal" href="../glossary.html#term-scorer"><span class="xref std std-term">scorer</span></a>, or just the
training or validation loss. By default, early-stopping is performed using
the default <a class="reference internal" href="../glossary.html#term-scorer"><span class="xref std std-term">scorer</span></a> of the estimator on a validation set but it is
also possible to perform early-stopping based on the loss value, which is
significantly faster.</p>
</div>
<div class="section" id="missing-values-support">
<h3>1.11.5.2. Missing values support<a class="headerlink" href="#missing-values-support" title="Permalink to this headline">¶</a></h3>
<p><a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingClassifier.html#sklearn.ensemble.HistGradientBoostingClassifier" title="sklearn.ensemble.HistGradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingClassifier</span></code></a> and
<a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingRegressor.html#sklearn.ensemble.HistGradientBoostingRegressor" title="sklearn.ensemble.HistGradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingRegressor</span></code></a> have built-in support for missing
values (NaNs).</p>
<p>During training, the tree grower learns at each split point whether samples
with missing values should go to the left or right child, based on the
potential gain. When predicting, samples with missing values are assigned to
the left or right child consequently:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.experimental</span> <span class="kn">import</span> <span class="n">enable_hist_gradient_boosting</span>  <span class="c1"># noqa</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">HistGradientBoostingClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">nan</span><span class="p">])</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">gbdt</span> <span class="o">=</span> <span class="n">HistGradientBoostingClassifier</span><span class="p">(</span><span class="n">min_samples_leaf</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">gbdt</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
<span class="go">array([0, 0, 1, 1])</span>
</pre></div>
</div>
<p>When the missingness pattern is predictive, the splits can be done on
whether the feature value is missing or not:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">nan</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">nan</span><span class="p">])</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">gbdt</span> <span class="o">=</span> <span class="n">HistGradientBoostingClassifier</span><span class="p">(</span><span class="n">min_samples_leaf</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
<span class="gp">... </span>                                      <span class="n">max_depth</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
<span class="gp">... </span>                                      <span class="n">learning_rate</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
<span class="gp">... </span>                                      <span class="n">max_iter</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">gbdt</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
<span class="go">array([0, 1, 0, 0, 1])</span>
</pre></div>
</div>
<p>If no missing values were encountered for a given feature during training,
then samples with missing values are mapped to whichever child has the most
samples.</p>
</div>
<div class="section" id="low-level-parallelism">
<h3>1.11.5.3. Low-level parallelism<a class="headerlink" href="#low-level-parallelism" title="Permalink to this headline">¶</a></h3>
<p><a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingClassifier.html#sklearn.ensemble.HistGradientBoostingClassifier" title="sklearn.ensemble.HistGradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingClassifier</span></code></a> and
<a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingRegressor.html#sklearn.ensemble.HistGradientBoostingRegressor" title="sklearn.ensemble.HistGradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingRegressor</span></code></a> have implementations that use OpenMP
for parallelization through Cython. For more details on how to control the
number of threads, please refer to our <a class="reference internal" href="computing.html#parallelism"><span class="std std-ref">Parallelism</span></a> notes.</p>
<p>The following parts are parallelized:</p>
<ul class="simple">
<li><p>mapping samples from real values to integer-valued bins (finding the bin
thresholds is however sequential)</p></li>
<li><p>building histograms is parallelized over features</p></li>
<li><p>finding the best split point at a node is parallelized over features</p></li>
<li><p>during fit, mapping samples into the left and right children is
parallelized over samples</p></li>
<li><p>gradient and hessians computations are parallelized over samples</p></li>
<li><p>predicting is parallelized over samples</p></li>
</ul>
</div>
<div class="section" id="why-it-s-faster">
<h3>1.11.5.4. Why it’s faster<a class="headerlink" href="#why-it-s-faster" title="Permalink to this headline">¶</a></h3>
<p>The bottleneck of a gradient boosting procedure is building the decision
trees. Building a traditional decision tree (as in the other GBDTs
<a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingClassifier.html#sklearn.ensemble.GradientBoostingClassifier" title="sklearn.ensemble.GradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingClassifier</span></code></a> and <a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingRegressor.html#sklearn.ensemble.GradientBoostingRegressor" title="sklearn.ensemble.GradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingRegressor</span></code></a>)
requires sorting the samples at each node (for
each feature). Sorting is needed so that the potential gain of a split point
can be computed efficiently. Splitting a single node has thus a complexity
of <span class="math notranslate nohighlight">\(\mathcal{O}(n_\text{features} \times n \log(n))\)</span> where <span class="math notranslate nohighlight">\(n\)</span>
is the number of samples at the node.</p>
<p><a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingClassifier.html#sklearn.ensemble.HistGradientBoostingClassifier" title="sklearn.ensemble.HistGradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingClassifier</span></code></a> and
<a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingRegressor.html#sklearn.ensemble.HistGradientBoostingRegressor" title="sklearn.ensemble.HistGradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingRegressor</span></code></a>, in contrast, do not require sorting the
feature values and instead use a data-structure called a histogram, where the
samples are implicitly ordered. Building a histogram has a
<span class="math notranslate nohighlight">\(\mathcal{O}(n)\)</span> complexity, so the node splitting procedure has a
<span class="math notranslate nohighlight">\(\mathcal{O}(n_\text{features} \times n)\)</span> complexity, much smaller
than the previous one. In addition, instead of considering <span class="math notranslate nohighlight">\(n\)</span> split
points, we here consider only <code class="docutils literal notranslate"><span class="pre">max_bins</span></code> split points, which is much
smaller.</p>
<p>In order to build histograms, the input data <code class="docutils literal notranslate"><span class="pre">X</span></code> needs to be binned into
integer-valued bins. This binning procedure does require sorting the feature
values, but it only happens once at the very beginning of the boosting process
(not at each node, like in <a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingClassifier.html#sklearn.ensemble.GradientBoostingClassifier" title="sklearn.ensemble.GradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingClassifier</span></code></a> and
<a class="reference internal" href="generated/sklearn.ensemble.GradientBoostingRegressor.html#sklearn.ensemble.GradientBoostingRegressor" title="sklearn.ensemble.GradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">GradientBoostingRegressor</span></code></a>).</p>
<p>Finally, many parts of the implementation of
<a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingClassifier.html#sklearn.ensemble.HistGradientBoostingClassifier" title="sklearn.ensemble.HistGradientBoostingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingClassifier</span></code></a> and
<a class="reference internal" href="generated/sklearn.ensemble.HistGradientBoostingRegressor.html#sklearn.ensemble.HistGradientBoostingRegressor" title="sklearn.ensemble.HistGradientBoostingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">HistGradientBoostingRegressor</span></code></a> are parallelized.</p>
<div class="topic">
<p class="topic-title">References</p>
<dl class="citation">
<dt class="label" id="f1999"><span class="brackets"><a class="fn-backref" href="#id20">F1999</a></span></dt>
<dd><p>Friedmann, Jerome H., 2007, <a class="reference external" href="https://statweb.stanford.edu/~jhf/ftp/stobst.pdf">“Stochastic Gradient Boosting”</a></p>
</dd>
<dt class="label" id="r2007"><span class="brackets"><a class="fn-backref" href="#id19">R2007</a></span></dt>
<dd><p>G. Ridgeway, “Generalized Boosted Models: A guide to the gbm
package”, 2007</p>
</dd>
<dt class="label" id="xgboost"><span class="brackets"><a class="fn-backref" href="#id24">XGBoost</a></span></dt>
<dd><p>Tianqi Chen, Carlos Guestrin, <a class="reference external" href="https://arxiv.org/abs/1603.02754">“XGBoost: A Scalable Tree
Boosting System”</a></p>
</dd>
<dt class="label" id="lightgbm"><span class="brackets">LightGBM</span><span class="fn-backref">(<a href="#id14">1</a>,<a href="#id22">2</a>)</span></dt>
<dd><p>Ke et. al. <a class="reference external" href="https://papers.nips.cc/paper/6907-lightgbm-a-highly-efficient-gradient-boosting-decision-tree">“LightGBM: A Highly Efficient Gradient
BoostingDecision Tree”</a></p>
</dd>
</dl>
</div>
</div>
</div>
<div class="section" id="voting-classifier">
<span id="id25"></span><h2>1.11.6. Voting Classifier<a class="headerlink" href="#voting-classifier" title="Permalink to this headline">¶</a></h2>
<p>The idea behind the <a class="reference internal" href="generated/sklearn.ensemble.VotingClassifier.html#sklearn.ensemble.VotingClassifier" title="sklearn.ensemble.VotingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">VotingClassifier</span></code></a> is to combine
conceptually different machine learning classifiers and use a majority vote
or the average predicted probabilities (soft vote) to predict the class labels.
Such a classifier can be useful for a set of equally well performing model
in order to balance out their individual weaknesses.</p>
<div class="section" id="majority-class-labels-majority-hard-voting">
<h3>1.11.6.1. Majority Class Labels (Majority/Hard Voting)<a class="headerlink" href="#majority-class-labels-majority-hard-voting" title="Permalink to this headline">¶</a></h3>
<p>In majority voting, the predicted class label for a particular sample is
the class label that represents the majority (mode) of the class labels
predicted by each individual classifier.</p>
<p>E.g., if the prediction for a given sample is</p>
<ul class="simple">
<li><p>classifier 1 -&gt; class 1</p></li>
<li><p>classifier 2 -&gt; class 1</p></li>
<li><p>classifier 3 -&gt; class 2</p></li>
</ul>
<p>the VotingClassifier (with <code class="docutils literal notranslate"><span class="pre">voting='hard'</span></code>) would classify the sample
as “class 1” based on the majority class label.</p>
<p>In the cases of a tie, the <a class="reference internal" href="generated/sklearn.ensemble.VotingClassifier.html#sklearn.ensemble.VotingClassifier" title="sklearn.ensemble.VotingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">VotingClassifier</span></code></a> will select the class
based on the ascending sort order. E.g., in the following scenario</p>
<ul class="simple">
<li><p>classifier 1 -&gt; class 2</p></li>
<li><p>classifier 2 -&gt; class 1</p></li>
</ul>
<p>the class label 1 will be assigned to the sample.</p>
<div class="section" id="id26">
<h4>1.11.6.1.1. Usage<a class="headerlink" href="#id26" title="Permalink to this headline">¶</a></h4>
<p>The following example shows how to fit the majority rule classifier:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">datasets</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">cross_val_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">LogisticRegression</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.naive_bayes</span> <span class="kn">import</span> <span class="n">GaussianNB</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">RandomForestClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">VotingClassifier</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">iris</span> <span class="o">=</span> <span class="n">datasets</span><span class="o">.</span><span class="n">load_iris</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">iris</span><span class="o">.</span><span class="n">data</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">:</span><span class="mi">3</span><span class="p">],</span> <span class="n">iris</span><span class="o">.</span><span class="n">target</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">clf1</span> <span class="o">=</span> <span class="n">LogisticRegression</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf2</span> <span class="o">=</span> <span class="n">RandomForestClassifier</span><span class="p">(</span><span class="n">n_estimators</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf3</span> <span class="o">=</span> <span class="n">GaussianNB</span><span class="p">()</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">eclf</span> <span class="o">=</span> <span class="n">VotingClassifier</span><span class="p">(</span>
<span class="gp">... </span>    <span class="n">estimators</span><span class="o">=</span><span class="p">[(</span><span class="s1">&#39;lr&#39;</span><span class="p">,</span> <span class="n">clf1</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;rf&#39;</span><span class="p">,</span> <span class="n">clf2</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;gnb&#39;</span><span class="p">,</span> <span class="n">clf3</span><span class="p">)],</span>
<span class="gp">... </span>    <span class="n">voting</span><span class="o">=</span><span class="s1">&#39;hard&#39;</span><span class="p">)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">clf</span><span class="p">,</span> <span class="n">label</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">([</span><span class="n">clf1</span><span class="p">,</span> <span class="n">clf2</span><span class="p">,</span> <span class="n">clf3</span><span class="p">,</span> <span class="n">eclf</span><span class="p">],</span> <span class="p">[</span><span class="s1">&#39;Logistic Regression&#39;</span><span class="p">,</span> <span class="s1">&#39;Random Forest&#39;</span><span class="p">,</span> <span class="s1">&#39;naive Bayes&#39;</span><span class="p">,</span> <span class="s1">&#39;Ensemble&#39;</span><span class="p">]):</span>
<span class="gp">... </span>    <span class="n">scores</span> <span class="o">=</span> <span class="n">cross_val_score</span><span class="p">(</span><span class="n">clf</span><span class="p">,</span> <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">scoring</span><span class="o">=</span><span class="s1">&#39;accuracy&#39;</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
<span class="gp">... </span>    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Accuracy: </span><span class="si">%0.2f</span><span class="s2"> (+/- </span><span class="si">%0.2f</span><span class="s2">) [</span><span class="si">%s</span><span class="s2">]&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">scores</span><span class="o">.</span><span class="n">mean</span><span class="p">(),</span> <span class="n">scores</span><span class="o">.</span><span class="n">std</span><span class="p">(),</span> <span class="n">label</span><span class="p">))</span>
<span class="go">Accuracy: 0.95 (+/- 0.04) [Logistic Regression]</span>
<span class="go">Accuracy: 0.94 (+/- 0.04) [Random Forest]</span>
<span class="go">Accuracy: 0.91 (+/- 0.04) [naive Bayes]</span>
<span class="go">Accuracy: 0.95 (+/- 0.04) [Ensemble]</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="weighted-average-probabilities-soft-voting">
<h3>1.11.6.2. Weighted Average Probabilities (Soft Voting)<a class="headerlink" href="#weighted-average-probabilities-soft-voting" title="Permalink to this headline">¶</a></h3>
<p>In contrast to majority voting (hard voting), soft voting
returns the class label as argmax of the sum of predicted probabilities.</p>
<p>Specific weights can be assigned to each classifier via the <code class="docutils literal notranslate"><span class="pre">weights</span></code>
parameter. When weights are provided, the predicted class probabilities
for each classifier are collected, multiplied by the classifier weight,
and averaged. The final class label is then derived from the class label
with the highest average probability.</p>
<p>To illustrate this with a simple example, let’s assume we have 3
classifiers and a 3-class classification problems where we assign
equal weights to all classifiers: w1=1, w2=1, w3=1.</p>
<p>The weighted average probabilities for a sample would then be
calculated as follows:</p>
<table class="docutils align-default">
<colgroup>
<col style="width: 35%" />
<col style="width: 22%" />
<col style="width: 22%" />
<col style="width: 22%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"><p>classifier</p></th>
<th class="head"><p>class 1</p></th>
<th class="head"><p>class 2</p></th>
<th class="head"><p>class 3</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>classifier 1</p></td>
<td><p>w1 * 0.2</p></td>
<td><p>w1 * 0.5</p></td>
<td><p>w1 * 0.3</p></td>
</tr>
<tr class="row-odd"><td><p>classifier 2</p></td>
<td><p>w2 * 0.6</p></td>
<td><p>w2 * 0.3</p></td>
<td><p>w2 * 0.1</p></td>
</tr>
<tr class="row-even"><td><p>classifier 3</p></td>
<td><p>w3 * 0.3</p></td>
<td><p>w3 * 0.4</p></td>
<td><p>w3 * 0.3</p></td>
</tr>
<tr class="row-odd"><td><p>weighted average</p></td>
<td><p>0.37</p></td>
<td><p>0.4</p></td>
<td><p>0.23</p></td>
</tr>
</tbody>
</table>
<p>Here, the predicted class label is 2, since it has the
highest average probability.</p>
<p>The following example illustrates how the decision regions may change
when a soft <a class="reference internal" href="generated/sklearn.ensemble.VotingClassifier.html#sklearn.ensemble.VotingClassifier" title="sklearn.ensemble.VotingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">VotingClassifier</span></code></a> is used based on an linear Support
Vector Machine, a Decision Tree, and a K-nearest neighbor classifier:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">datasets</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.tree</span> <span class="kn">import</span> <span class="n">DecisionTreeClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.neighbors</span> <span class="kn">import</span> <span class="n">KNeighborsClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.svm</span> <span class="kn">import</span> <span class="n">SVC</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">itertools</span> <span class="kn">import</span> <span class="n">product</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">VotingClassifier</span>

<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Loading some example data</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">iris</span> <span class="o">=</span> <span class="n">datasets</span><span class="o">.</span><span class="n">load_iris</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span> <span class="o">=</span> <span class="n">iris</span><span class="o">.</span><span class="n">data</span><span class="p">[:,</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span> <span class="o">=</span> <span class="n">iris</span><span class="o">.</span><span class="n">target</span>

<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Training classifiers</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf1</span> <span class="o">=</span> <span class="n">DecisionTreeClassifier</span><span class="p">(</span><span class="n">max_depth</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf2</span> <span class="o">=</span> <span class="n">KNeighborsClassifier</span><span class="p">(</span><span class="n">n_neighbors</span><span class="o">=</span><span class="mi">7</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf3</span> <span class="o">=</span> <span class="n">SVC</span><span class="p">(</span><span class="n">kernel</span><span class="o">=</span><span class="s1">&#39;rbf&#39;</span><span class="p">,</span> <span class="n">probability</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">eclf</span> <span class="o">=</span> <span class="n">VotingClassifier</span><span class="p">(</span><span class="n">estimators</span><span class="o">=</span><span class="p">[(</span><span class="s1">&#39;dt&#39;</span><span class="p">,</span> <span class="n">clf1</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;knn&#39;</span><span class="p">,</span> <span class="n">clf2</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;svc&#39;</span><span class="p">,</span> <span class="n">clf3</span><span class="p">)],</span>
<span class="gp">... </span>                        <span class="n">voting</span><span class="o">=</span><span class="s1">&#39;soft&#39;</span><span class="p">,</span> <span class="n">weights</span><span class="o">=</span><span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">clf1</span> <span class="o">=</span> <span class="n">clf1</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf2</span> <span class="o">=</span> <span class="n">clf2</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf3</span> <span class="o">=</span> <span class="n">clf3</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">eclf</span> <span class="o">=</span> <span class="n">eclf</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
</pre></div>
</div>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/ensemble/plot_voting_decision_regions.html"><img alt="modules/../auto_examples/ensemble/images/sphx_glr_plot_voting_decision_regions_001.png" src="modules/../auto_examples/ensemble/images/sphx_glr_plot_voting_decision_regions_001.png" /></a>
</div>
</div>
<div class="section" id="using-the-votingclassifier-with-gridsearchcv">
<h3>1.11.6.3. Using the <code class="docutils literal notranslate"><span class="pre">VotingClassifier</span></code> with <code class="docutils literal notranslate"><span class="pre">GridSearchCV</span></code><a class="headerlink" href="#using-the-votingclassifier-with-gridsearchcv" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.ensemble.VotingClassifier.html#sklearn.ensemble.VotingClassifier" title="sklearn.ensemble.VotingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">VotingClassifier</span></code></a> can also be used together with
<a class="reference internal" href="generated/sklearn.model_selection.GridSearchCV.html#sklearn.model_selection.GridSearchCV" title="sklearn.model_selection.GridSearchCV"><code class="xref py py-class docutils literal notranslate"><span class="pre">GridSearchCV</span></code></a> in order to tune the
hyperparameters of the individual estimators:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">GridSearchCV</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf1</span> <span class="o">=</span> <span class="n">LogisticRegression</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf2</span> <span class="o">=</span> <span class="n">RandomForestClassifier</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf3</span> <span class="o">=</span> <span class="n">GaussianNB</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">eclf</span> <span class="o">=</span> <span class="n">VotingClassifier</span><span class="p">(</span>
<span class="gp">... </span>    <span class="n">estimators</span><span class="o">=</span><span class="p">[(</span><span class="s1">&#39;lr&#39;</span><span class="p">,</span> <span class="n">clf1</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;rf&#39;</span><span class="p">,</span> <span class="n">clf2</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;gnb&#39;</span><span class="p">,</span> <span class="n">clf3</span><span class="p">)],</span>
<span class="gp">... </span>    <span class="n">voting</span><span class="o">=</span><span class="s1">&#39;soft&#39;</span>
<span class="gp">... </span><span class="p">)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">params</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;lr__C&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">100.0</span><span class="p">],</span> <span class="s1">&#39;rf__n_estimators&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">20</span><span class="p">,</span> <span class="mi">200</span><span class="p">]}</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">grid</span> <span class="o">=</span> <span class="n">GridSearchCV</span><span class="p">(</span><span class="n">estimator</span><span class="o">=</span><span class="n">eclf</span><span class="p">,</span> <span class="n">param_grid</span><span class="o">=</span><span class="n">params</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">grid</span> <span class="o">=</span> <span class="n">grid</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">iris</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="n">iris</span><span class="o">.</span><span class="n">target</span><span class="p">)</span>
</pre></div>
</div>
<div class="section" id="id27">
<h4>1.11.6.3.1. Usage<a class="headerlink" href="#id27" title="Permalink to this headline">¶</a></h4>
<p>In order to predict the class labels based on the predicted
class-probabilities (scikit-learn estimators in the VotingClassifier
must support <code class="docutils literal notranslate"><span class="pre">predict_proba</span></code> method):</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">eclf</span> <span class="o">=</span> <span class="n">VotingClassifier</span><span class="p">(</span>
<span class="gp">... </span>    <span class="n">estimators</span><span class="o">=</span><span class="p">[(</span><span class="s1">&#39;lr&#39;</span><span class="p">,</span> <span class="n">clf1</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;rf&#39;</span><span class="p">,</span> <span class="n">clf2</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;gnb&#39;</span><span class="p">,</span> <span class="n">clf3</span><span class="p">)],</span>
<span class="gp">... </span>    <span class="n">voting</span><span class="o">=</span><span class="s1">&#39;soft&#39;</span>
<span class="gp">... </span><span class="p">)</span>
</pre></div>
</div>
<p>Optionally, weights can be provided for the individual classifiers:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">eclf</span> <span class="o">=</span> <span class="n">VotingClassifier</span><span class="p">(</span>
<span class="gp">... </span>    <span class="n">estimators</span><span class="o">=</span><span class="p">[(</span><span class="s1">&#39;lr&#39;</span><span class="p">,</span> <span class="n">clf1</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;rf&#39;</span><span class="p">,</span> <span class="n">clf2</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;gnb&#39;</span><span class="p">,</span> <span class="n">clf3</span><span class="p">)],</span>
<span class="gp">... </span>    <span class="n">voting</span><span class="o">=</span><span class="s1">&#39;soft&#39;</span><span class="p">,</span> <span class="n">weights</span><span class="o">=</span><span class="p">[</span><span class="mi">2</span><span class="p">,</span><span class="mi">5</span><span class="p">,</span><span class="mi">1</span><span class="p">]</span>
<span class="gp">... </span><span class="p">)</span>
</pre></div>
</div>
</div>
</div>
</div>
<div class="section" id="voting-regressor">
<span id="id28"></span><h2>1.11.7. Voting Regressor<a class="headerlink" href="#voting-regressor" title="Permalink to this headline">¶</a></h2>
<p>The idea behind the <a class="reference internal" href="generated/sklearn.ensemble.VotingRegressor.html#sklearn.ensemble.VotingRegressor" title="sklearn.ensemble.VotingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">VotingRegressor</span></code></a> is to combine conceptually
different machine learning regressors and return the average predicted values.
Such a regressor can be useful for a set of equally well performing models
in order to balance out their individual weaknesses.</p>
<div class="section" id="id29">
<h3>1.11.7.1. Usage<a class="headerlink" href="#id29" title="Permalink to this headline">¶</a></h3>
<p>The following example shows how to fit the VotingRegressor:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">load_boston</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">GradientBoostingRegressor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">RandomForestRegressor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">LinearRegression</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">VotingRegressor</span>

<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Loading some example data</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">load_boston</span><span class="p">(</span><span class="n">return_X_y</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Training classifiers</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">reg1</span> <span class="o">=</span> <span class="n">GradientBoostingRegressor</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_estimators</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">reg2</span> <span class="o">=</span> <span class="n">RandomForestRegressor</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_estimators</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">reg3</span> <span class="o">=</span> <span class="n">LinearRegression</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">ereg</span> <span class="o">=</span> <span class="n">VotingRegressor</span><span class="p">(</span><span class="n">estimators</span><span class="o">=</span><span class="p">[(</span><span class="s1">&#39;gb&#39;</span><span class="p">,</span> <span class="n">reg1</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;rf&#39;</span><span class="p">,</span> <span class="n">reg2</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;lr&#39;</span><span class="p">,</span> <span class="n">reg3</span><span class="p">)])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">ereg</span> <span class="o">=</span> <span class="n">ereg</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
</pre></div>
</div>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/ensemble/plot_voting_regressor.html"><img alt="modules/../auto_examples/ensemble/images/sphx_glr_plot_voting_regressor_001.png" src="modules/../auto_examples/ensemble/images/sphx_glr_plot_voting_regressor_001.png" /></a>
</div>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p><a class="reference internal" href="../auto_examples/ensemble/plot_voting_regressor.html#sphx-glr-auto-examples-ensemble-plot-voting-regressor-py"><span class="std std-ref">Plot individual and voting regression predictions</span></a></p></li>
</ul>
</div>
</div>
</div>
<div class="section" id="stacked-generalization">
<span id="stacking"></span><h2>1.11.8. Stacked generalization<a class="headerlink" href="#stacked-generalization" title="Permalink to this headline">¶</a></h2>
<p>Stacked generalization is a method for combining estimators to reduce their
biases <a class="reference internal" href="#w1992" id="id30"><span>[W1992]</span></a> <a class="reference internal" href="#htf" id="id31"><span>[HTF]</span></a>. More precisely, the predictions of each individual
estimator are stacked together and used as input to a final estimator to
compute the prediction. This final estimator is trained through
cross-validation.</p>
<p>The <a class="reference internal" href="generated/sklearn.ensemble.StackingClassifier.html#sklearn.ensemble.StackingClassifier" title="sklearn.ensemble.StackingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">StackingClassifier</span></code></a> and <a class="reference internal" href="generated/sklearn.ensemble.StackingRegressor.html#sklearn.ensemble.StackingRegressor" title="sklearn.ensemble.StackingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">StackingRegressor</span></code></a> provide such
strategies which can be applied to classification and regression problems.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">estimators</span></code> parameter corresponds to the list of the estimators which
are stacked together in parallel on the input data. It should be given as a
list of names and estimators:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">RidgeCV</span><span class="p">,</span> <span class="n">LassoCV</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.svm</span> <span class="kn">import</span> <span class="n">SVR</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">estimators</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">&#39;ridge&#39;</span><span class="p">,</span> <span class="n">RidgeCV</span><span class="p">()),</span>
<span class="gp">... </span>              <span class="p">(</span><span class="s1">&#39;lasso&#39;</span><span class="p">,</span> <span class="n">LassoCV</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">42</span><span class="p">)),</span>
<span class="gp">... </span>              <span class="p">(</span><span class="s1">&#39;svr&#39;</span><span class="p">,</span> <span class="n">SVR</span><span class="p">(</span><span class="n">C</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">))]</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">final_estimator</span></code> will use the predictions of the <code class="docutils literal notranslate"><span class="pre">estimators</span></code> as input. It
needs to be a classifier or a regressor when using <a class="reference internal" href="generated/sklearn.ensemble.StackingClassifier.html#sklearn.ensemble.StackingClassifier" title="sklearn.ensemble.StackingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">StackingClassifier</span></code></a>
or <a class="reference internal" href="generated/sklearn.ensemble.StackingRegressor.html#sklearn.ensemble.StackingRegressor" title="sklearn.ensemble.StackingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">StackingRegressor</span></code></a>, respectively:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">GradientBoostingRegressor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">StackingRegressor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">reg</span> <span class="o">=</span> <span class="n">StackingRegressor</span><span class="p">(</span>
<span class="gp">... </span>    <span class="n">estimators</span><span class="o">=</span><span class="n">estimators</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">final_estimator</span><span class="o">=</span><span class="n">GradientBoostingRegressor</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">42</span><span class="p">))</span>
</pre></div>
</div>
<p>To train the <code class="docutils literal notranslate"><span class="pre">estimators</span></code> and <code class="docutils literal notranslate"><span class="pre">final_estimator</span></code>, the <code class="docutils literal notranslate"><span class="pre">fit</span></code> method needs
to be called on the training data:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">load_boston</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">load_boston</span><span class="p">(</span><span class="n">return_X_y</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span>
<span class="gp">... </span>                                                    <span class="n">random_state</span><span class="o">=</span><span class="mi">42</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">reg</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
<span class="go">StackingRegressor(...)</span>
</pre></div>
</div>
<p>During training, the <code class="docutils literal notranslate"><span class="pre">estimators</span></code> are fitted on the whole training data
<code class="docutils literal notranslate"><span class="pre">X_train</span></code>. They will be used when calling <code class="docutils literal notranslate"><span class="pre">predict</span></code> or <code class="docutils literal notranslate"><span class="pre">predict_proba</span></code>. To
generalize and avoid over-fitting, the <code class="docutils literal notranslate"><span class="pre">final_estimator</span></code> is trained on
out-samples using <a class="reference internal" href="generated/sklearn.model_selection.cross_val_predict.html#sklearn.model_selection.cross_val_predict" title="sklearn.model_selection.cross_val_predict"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.model_selection.cross_val_predict</span></code></a> internally.</p>
<p>For <a class="reference internal" href="generated/sklearn.ensemble.StackingClassifier.html#sklearn.ensemble.StackingClassifier" title="sklearn.ensemble.StackingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">StackingClassifier</span></code></a>, note that the output of the <code class="docutils literal notranslate"><span class="pre">estimators</span></code> is
controlled by the parameter <code class="docutils literal notranslate"><span class="pre">stack_method</span></code> and it is called by each estimator.
This parameter is either a string, being estimator method names, or <code class="docutils literal notranslate"><span class="pre">'auto'</span></code>
which will automatically identify an available method depending on the
availability, tested in the order of preference: <code class="docutils literal notranslate"><span class="pre">predict_proba</span></code>,
<code class="docutils literal notranslate"><span class="pre">decision_function</span></code> and <code class="docutils literal notranslate"><span class="pre">predict</span></code>.</p>
<p>A <a class="reference internal" href="generated/sklearn.ensemble.StackingRegressor.html#sklearn.ensemble.StackingRegressor" title="sklearn.ensemble.StackingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">StackingRegressor</span></code></a> and <a class="reference internal" href="generated/sklearn.ensemble.StackingClassifier.html#sklearn.ensemble.StackingClassifier" title="sklearn.ensemble.StackingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">StackingClassifier</span></code></a> can be used as
any other regressor or classifier, exposing a <code class="docutils literal notranslate"><span class="pre">predict</span></code>, <code class="docutils literal notranslate"><span class="pre">predict_proba</span></code>, and
<code class="docutils literal notranslate"><span class="pre">decision_function</span></code> methods, e.g.:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="n">reg</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_test</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">r2_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="s1">&#39;R2 score: </span><span class="si">{:.2f}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">r2_score</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)))</span>
<span class="go">R2 score: 0.81</span>
</pre></div>
</div>
<p>Note that it is also possible to get the output of the stacked outputs of the
<code class="docutils literal notranslate"><span class="pre">estimators</span></code> using the <code class="docutils literal notranslate"><span class="pre">transform</span></code> method:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">reg</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">X_test</span><span class="p">[:</span><span class="mi">5</span><span class="p">])</span>
<span class="go">array([[28.78..., 28.43...  , 22.62...],</span>
<span class="go">       [35.96..., 32.58..., 23.68...],</span>
<span class="go">       [14.97..., 14.05..., 16.45...],</span>
<span class="go">       [25.19..., 25.54..., 22.92...],</span>
<span class="go">       [18.93..., 19.26..., 17.03... ]])</span>
</pre></div>
</div>
<p>In practise, a stacking predictor predict as good as the best predictor of the
base layer and even sometimes outputperform it by combining the different
strength of the these predictors. However, training a stacking predictor is
computationally expensive.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>For <a class="reference internal" href="generated/sklearn.ensemble.StackingClassifier.html#sklearn.ensemble.StackingClassifier" title="sklearn.ensemble.StackingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">StackingClassifier</span></code></a>, when using <code class="docutils literal notranslate"><span class="pre">stack_method_='predict_proba'</span></code>,
the first column is dropped when the problem is a binary classification
problem. Indeed, both probability columns predicted by each estimator are
perfectly collinear.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Multiple stacking layers can be achieved by assigning <code class="docutils literal notranslate"><span class="pre">final_estimator</span></code> to
a <a class="reference internal" href="generated/sklearn.ensemble.StackingClassifier.html#sklearn.ensemble.StackingClassifier" title="sklearn.ensemble.StackingClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">StackingClassifier</span></code></a> or <a class="reference internal" href="generated/sklearn.ensemble.StackingRegressor.html#sklearn.ensemble.StackingRegressor" title="sklearn.ensemble.StackingRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">StackingRegressor</span></code></a>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">final_layer</span> <span class="o">=</span> <span class="n">StackingRegressor</span><span class="p">(</span>
<span class="gp">... </span>    <span class="n">estimators</span><span class="o">=</span><span class="p">[(</span><span class="s1">&#39;rf&#39;</span><span class="p">,</span> <span class="n">RandomForestRegressor</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">42</span><span class="p">)),</span>
<span class="gp">... </span>                <span class="p">(</span><span class="s1">&#39;gbrt&#39;</span><span class="p">,</span> <span class="n">GradientBoostingRegressor</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">42</span><span class="p">))],</span>
<span class="gp">... </span>    <span class="n">final_estimator</span><span class="o">=</span><span class="n">RidgeCV</span><span class="p">()</span>
<span class="gp">... </span>    <span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">multi_layer_regressor</span> <span class="o">=</span> <span class="n">StackingRegressor</span><span class="p">(</span>
<span class="gp">... </span>    <span class="n">estimators</span><span class="o">=</span><span class="p">[(</span><span class="s1">&#39;ridge&#39;</span><span class="p">,</span> <span class="n">RidgeCV</span><span class="p">()),</span>
<span class="gp">... </span>                <span class="p">(</span><span class="s1">&#39;lasso&#39;</span><span class="p">,</span> <span class="n">LassoCV</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">42</span><span class="p">)),</span>
<span class="gp">... </span>                <span class="p">(</span><span class="s1">&#39;svr&#39;</span><span class="p">,</span> <span class="n">SVR</span><span class="p">(</span><span class="n">C</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">,</span> <span class="n">kernel</span><span class="o">=</span><span class="s1">&#39;rbf&#39;</span><span class="p">))],</span>
<span class="gp">... </span>    <span class="n">final_estimator</span><span class="o">=</span><span class="n">final_layer</span>
<span class="gp">... </span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">multi_layer_regressor</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
<span class="go">StackingRegressor(...)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="s1">&#39;R2 score: </span><span class="si">{:.2f}</span><span class="s1">&#39;</span>
<span class="gp">... </span>      <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">multi_layer_regressor</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)))</span>
<span class="go">R2 score: 0.82</span>
</pre></div>
</div>
</div>
<div class="topic">
<p class="topic-title">References</p>
<dl class="citation">
<dt class="label" id="w1992"><span class="brackets"><a class="fn-backref" href="#id30">W1992</a></span></dt>
<dd><p>Wolpert, David H. “Stacked generalization.” Neural networks 5.2
(1992): 241-259.</p>
</dd>
</dl>
</div>
</div>
</div>


      </div>
    <div class="container">
      <footer class="sk-content-footer">
            &copy; 2007 - 2019, scikit-learn developers (BSD License).
          <a href="../_sources/modules/ensemble.rst.txt" rel="nofollow">Show this page source</a>
      </footer>
    </div>
  </div>
</div>
<script src="../_static/js/vendor/bootstrap.min.js"></script>

<script>
    window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
    ga('create', 'UA-22606712-2', 'auto');
    ga('set', 'anonymizeIp', true);
    ga('send', 'pageview');
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>


<script>
$(document).ready(function() {
    /* Add a [>>>] button on the top-right corner of code samples to hide
     * the >>> and ... prompts and the output and thus make the code
     * copyable. */
    var div = $('.highlight-python .highlight,' +
                '.highlight-python3 .highlight,' +
                '.highlight-pycon .highlight,' +
		'.highlight-default .highlight')
    var pre = div.find('pre');

    // get the styles from the current theme
    pre.parent().parent().css('position', 'relative');
    var hide_text = 'Hide prompts and outputs';
    var show_text = 'Show prompts and outputs';

    // create and add the button to all the code blocks that contain >>>
    div.each(function(index) {
        var jthis = $(this);
        if (jthis.find('.gp').length > 0) {
            var button = $('<span class="copybutton">&gt;&gt;&gt;</span>');
            button.attr('title', hide_text);
            button.data('hidden', 'false');
            jthis.prepend(button);
        }
        // tracebacks (.gt) contain bare text elements that need to be
        // wrapped in a span to work with .nextUntil() (see later)
        jthis.find('pre:has(.gt)').contents().filter(function() {
            return ((this.nodeType == 3) && (this.data.trim().length > 0));
        }).wrap('<span>');
    });

    // define the behavior of the button when it's clicked
    $('.copybutton').click(function(e){
        e.preventDefault();
        var button = $(this);
        if (button.data('hidden') === 'false') {
            // hide the code output
            button.parent().find('.go, .gp, .gt').hide();
            button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden');
            button.css('text-decoration', 'line-through');
            button.attr('title', show_text);
            button.data('hidden', 'true');
        } else {
            // show the code output
            button.parent().find('.go, .gp, .gt').show();
            button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible');
            button.css('text-decoration', 'none');
            button.attr('title', hide_text);
            button.data('hidden', 'false');
        }
    });

	/*** Add permalink buttons next to glossary terms ***/
	$('dl.glossary > dt[id]').append(function() {
		return ('<a class="headerlink" href="#' +
			    this.getAttribute('id') +
			    '" title="Permalink to this term">¶</a>');
	});
  /*** Hide navbar when scrolling down ***/
  // Returns true when headerlink target matches hash in url
  (function() {
    hashTargetOnTop = function() {
        var hash = window.location.hash;
        if ( hash.length < 2 ) { return false; }

        var target = document.getElementById( hash.slice(1) );
        if ( target === null ) { return false; }

        var top = target.getBoundingClientRect().top;
        return (top < 2) && (top > -2);
    };

    // Hide navbar on load if hash target is on top
    var navBar = document.getElementById("navbar");
    var navBarToggler = document.getElementById("sk-navbar-toggler");
    var navBarHeightHidden = "-" + navBar.getBoundingClientRect().height + "px";
    var $window = $(window);

    hideNavBar = function() {
        navBar.style.top = navBarHeightHidden;
    };

    showNavBar = function() {
        navBar.style.top = "0";
    }

    if (hashTargetOnTop()) {
        hideNavBar()
    }

    var prevScrollpos = window.pageYOffset;
    hideOnScroll = function(lastScrollTop) {
        if (($window.width() < 768) && (navBarToggler.getAttribute("aria-expanded") === 'true')) {
            return;
        }
        if (lastScrollTop > 2 && (prevScrollpos <= lastScrollTop) || hashTargetOnTop()){
            hideNavBar()
        } else {
            showNavBar()
        }
        prevScrollpos = lastScrollTop;
    };

    /*** high preformance scroll event listener***/
    var raf = window.requestAnimationFrame ||
        window.webkitRequestAnimationFrame ||
        window.mozRequestAnimationFrame ||
        window.msRequestAnimationFrame ||
        window.oRequestAnimationFrame;
    var lastScrollTop = $window.scrollTop();

    if (raf) {
        loop();
    }

    function loop() {
        var scrollTop = $window.scrollTop();
        if (lastScrollTop === scrollTop) {
            raf(loop);
            return;
        } else {
            lastScrollTop = scrollTop;
            hideOnScroll(lastScrollTop);
            raf(loop);
        }
    }
  })();
});

</script>
    
<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js"></script>
    
</body>
</html>